From 2f4c6900227f7b4da40dc4229459939288f66bff Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Mon, 14 Sep 2026 02:47:14 +0530 Subject: [PATCH 01/46] feat(docker-compose): Add Docker Compose configurations for local, dev, and master environments - Created docker-compose.dev.yaml for development environment with API server, worker, notification processor, delayed processor, recovery service, and admin dashboard. - Created docker-compose.infra.yaml for shared infrastructure services including MongoDB, Kafka, Redis, Loki, and Grafana. - Created docker-compose.local.yaml for local environment with build context for API server and dashboard. - Created docker-compose.master.yaml for master environment using latest images from GHCR. - Added scripts for switching between environments (switch-env.ps1 and switch-env.sh) with options for rebuilding images and pulling the latest versions. --- AGENTS.md | 2 +- CONTRIBUTING.md | 21 +- docker-compose.dev.yaml | 371 ------------------ docker-compose.yaml | 95 ----- docker-compose/README.md | 123 ++++++ docker-compose/docker-compose.dev.yaml | 190 +++++++++ .../docker-compose.infra.yaml | 88 +++-- docker-compose/docker-compose.local.yaml | 196 +++++++++ docker-compose/docker-compose.master.yaml | 190 +++++++++ docker-compose/scripts/switch-env.ps1 | 217 ++++++++++ docker-compose/scripts/switch-env.sh | 206 ++++++++++ 11 files changed, 1191 insertions(+), 508 deletions(-) delete mode 100644 docker-compose.dev.yaml delete mode 100644 docker-compose.yaml create mode 100644 docker-compose/README.md create mode 100644 docker-compose/docker-compose.dev.yaml rename docker-compose.infra.yaml => docker-compose/docker-compose.infra.yaml (58%) create mode 100644 docker-compose/docker-compose.local.yaml create mode 100644 docker-compose/docker-compose.master.yaml create mode 100644 docker-compose/scripts/switch-env.ps1 create mode 100644 docker-compose/scripts/switch-env.sh diff --git a/AGENTS.md b/AGENTS.md index d688c65..083f22a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,4 +32,4 @@ Use short Conventional Commit style messages such as `feat: add email retry poli PRs should include a summary, linked issue when applicable, test evidence, and screenshots or GIFs for `dashboard/` UI changes. Husky `pre-push` runs both lint commands. ## Security & Configuration Tips -Do not commit secrets. Use `.env.example` as the template for `.env`, and update it when adding configuration. For local full-stack work, prefer `docker-compose.dev.yaml`; for core-only work, start infrastructure with `docker compose -f docker-compose.infra.yaml up -d`. +Do not commit secrets. Use `.env.example` as the template for `.env`, and update it when adding configuration. For local full-stack work, use `./docker-compose/scripts/switch-env.ps1 -Env local -Rebuild` (or `docker compose -f docker-compose/docker-compose.local.yaml up -d`); for shared infrastructure, use `docker compose -f docker-compose/docker-compose.infra.yaml up -d` (or `./docker-compose/scripts/switch-env.ps1 -Env infra-only`). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3bce98e..d904281 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,7 @@ This repository contains: Recommended development paths: - Use `.devcontainer` (preferred for easy dev setup without any clutter). -- Use `docker-compose.dev.yaml` for the full stack. +- Use `docker-compose/` for full-stack Docker development with the environment switcher. - Run app code locally only if MongoDB, Redis, and Kafka are available. - Create the `simplens.config.yaml` file for any local development option below. - Refer the [Plugin Configuration Guide](https://www.simplens.in/docs/core/self-hosting#plugin-configuration) for manually creating `simplens.config.yaml` configuration file. @@ -40,18 +40,21 @@ Recommended development paths: 2. Let post-create scripts install dependencies automatically. 3. Start developing with local source mounted in the container. -### Option B: Docker Compose (`docker-compose.dev.yaml`) +### Option B: Docker Compose (`docker-compose/`) -Use when you want the full stack via Docker: +Use when you want the full stack via Docker with the environment switcher: ```bash -docker compose -f docker-compose.dev.yaml build -docker compose -f docker-compose.dev.yaml up -d +# Windows +./docker-compose/scripts/switch-env.ps1 -Env local -Rebuild + +# Linux/macOS +./docker-compose/scripts/switch-env.sh local --rebuild ``` Note: -- Code changes are not always reflected automatically in containerized runtime images. +- Code changes are rebuilt into the local image when passing `-Rebuild` or `--rebuild`. - Rebuild/restart relevant services after source changes when needed. ### Option C: Run Core Locally + Infra Services @@ -75,12 +78,12 @@ cp .env.example .env Copy-Item .env.example .env ``` -3. Start required infrastructure (MongoDB, Kafka, Redis) via Docker: +3. Start required infrastructure (MongoDB, Kafka, Redis, Loki, Grafana) via Docker: -- Set the env variable `INFRA_HOST=localhost` before starting the services in `docker-compose.infra.yaml` +- Set the env variable `INFRA_HOST=localhost` before starting the services in `docker-compose/docker-compose.infra.yaml` ```bash -docker compose -f docker-compose.infra.yaml up -d +docker compose -f docker-compose/docker-compose.infra.yaml up -d ``` ## Development Commands diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml deleted file mode 100644 index 019974f..0000000 --- a/docker-compose.dev.yaml +++ /dev/null @@ -1,371 +0,0 @@ -services: - # ============================================ - # Infrastructure Services - # ============================================ - mongo: - image: mongo:7.0 - container_name: mongo - command: [ "--replSet", "rs0", "--bind_ip_all", "--port", "27017" ] - ports: - - 27017:27017 - extra_hosts: - - "host.docker.internal:host-gateway" - healthcheck: - test: echo "try { rs.status() } catch (err) { rs.initiate({_id:'rs0',members:[{_id:0,host:'host.docker.internal:27017'}]}) }" | mongosh --port 27017 --quiet - interval: 5s - timeout: 30s - start_period: 0s - start_interval: 1s - retries: 30 - volumes: - - "mongo_data:/data/db" - - "mongo_config:/data/configdb" - networks: - - ns-network - - kafka: - image: apache/kafka-native - container_name: kafka - ports: - - "9092:9092" - environment: - # Configure listeners for both docker and host communication - KAFKA_LISTENERS: CONTROLLER://localhost:9091,HOST://0.0.0.0:9092,DOCKER://0.0.0.0:9093 - KAFKA_ADVERTISED_LISTENERS: HOST://localhost:9092,DOCKER://kafka:9093 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,DOCKER:PLAINTEXT,HOST:PLAINTEXT - - # Settings required for KRaft mode - KAFKA_NODE_ID: 1 - KAFKA_PROCESS_ROLES: broker,controller - KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9091 - - # Listener to use for broker-to-broker communication - KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER - - # Required for a single node cluster - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - - # Disable auto-topic creation - API server will create topics with correct partitions - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" - volumes: - - "kafka_data:/var/lib/kafka/data" - networks: - - ns-network - - kafka-ui: - image: kafbat/kafka-ui:main - container_name: kafka-ui - ports: - - 8080:8080 - environment: - DYNAMIC_CONFIG_ENABLED: "true" - KAFKA_CLUSTERS_0_NAME: local - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9093 - depends_on: - - kafka - networks: - - ns-network - - redis: - image: redis:7-alpine - container_name: redis - ports: - - "6379:6379" - command: redis-server --appendonly yes - volumes: - - "redis_data:/data" - networks: - - ns-network - healthcheck: - test: [ "CMD", "redis-cli", "ping" ] - interval: 5s - timeout: 3s - retries: 5 - - # ============================================ - # Observability Services - # ============================================ - - loki: - image: grafana/loki:2.9.0 - container_name: loki - ports: - - "3100:3100" - command: -config.file=/etc/loki/local-config.yaml - volumes: - - "loki_data:/loki" - networks: - - ns-network - healthcheck: - test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1" ] - interval: 10s - timeout: 5s - retries: 5 - - grafana: - image: grafana/grafana:10.2.0 - container_name: grafana - ports: - - "3001:3000" - environment: - - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - - GF_SECURITY_ADMIN_PASSWORD=admin - volumes: - - "grafana_data:/var/lib/grafana" - depends_on: - loki: - condition: service_healthy - networks: - - ns-network - - # ============================================ - # Application Services - # ============================================ - - # API Server - Handles HTTP requests for /notification and /notification/batch - api: - build: - context: . - dockerfile: Dockerfile - container_name: api - image: simplens-core:latest - ports: - - "3000:3000" - env_file: - - .env - environment: - NODE_ENV: ${NODE_ENV:-production} - PORT: ${PORT:-3000} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - NS_API_KEY: ${NS_API_KEY:-dev-api-key-change-in-production} - MAX_BATCH_REQ_LIMIT: ${MAX_BATCH_REQ_LIMIT:-1000} - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - DELAYED_PARTITION: ${DELAYED_PARTITION:-1} - NOTIFICATION_STATUS_PARTITION: ${NOTIFICATION_STATUS_PARTITION:-1} - SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} - volumes: - # Mount plugins directory for user-installed plugins (optional) - - plugin-data:/app/.plugins - - logs-data:/app/logs - - ./simplens.config.yaml:/app/simplens.config.yaml:ro - command: [ "node", "dist/api/server.js" ] - depends_on: - mongo: - condition: service_healthy - kafka: - condition: service_started - redis: - condition: service_healthy - loki: - condition: service_healthy - networks: - - ns-network - restart: unless-stopped - healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] - interval: 30s - timeout: 10s - retries: 3 - start_period: 10s - - # Background Worker - Polls outbox, publishes to Kafka, updates status in MongoDB - worker: - image: simplens-core:latest - extra_hosts: - - "host.docker.internal:host-gateway" - env_file: - - .env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - OUTBOX_POLL_INTERVAL_MS: ${OUTBOX_POLL_INTERVAL_MS:-5000} - OUTBOX_BATCH_SIZE: ${OUTBOX_BATCH_SIZE:-100} - OUTBOX_CLAIM_TIMEOUT_MS: ${OUTBOX_CLAIM_TIMEOUT_MS:-30000} - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - volumes: - - logs-data:/app/logs - command: [ "node", "dist/workers/worker.js" ] - depends_on: - mongo: - condition: service_healthy - kafka: - condition: service_started - redis: - condition: service_healthy - loki: - condition: service_healthy - api: - condition: service_healthy - networks: - - ns-network - restart: unless-stopped - - # Unified Notification Processor - Uses plugin system for all channels - notification-processor: - image: simplens-core:latest - env_file: - - .env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - # Plugin Configuration - path to simplens.config.yaml - SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} - # Processor Mode - 'all' for all channels, or specific channel name - PROCESSOR_CHANNEL: ${PROCESSOR_CHANNEL:-all} - # Retry Configuration - MAX_RETRY_COUNT: ${MAX_RETRY_COUNT:-5} - IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} - PROCESSING_TTL_SECONDS: ${PROCESSING_TTL_SECONDS:-120} - # Logging - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - command: [ "node", "dist/processors/unified/unified.processor.js" ] - volumes: - # Mount plugins directory for user-installed plugins (optional) - - plugin-data:/app/.plugins - - logs-data:/app/logs - - ./simplens.config.yaml:/app/simplens.config.yaml:ro - depends_on: - mongo: - condition: service_healthy - kafka: - condition: service_started - redis: - condition: service_healthy - loki: - condition: service_healthy - api: - condition: service_healthy - networks: - - ns-network - restart: unless-stopped - - # Delayed Processor - Handles scheduled notifications via Redis ZSET queue - delayed-processor: - image: simplens-core:latest - env_file: - - .env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - # Delayed Queue Configuration - DELAYED_POLL_INTERVAL_MS: ${DELAYED_POLL_INTERVAL_MS:-1000} - DELAYED_BATCH_SIZE: ${DELAYED_BATCH_SIZE:-10} - MAX_POLLER_RETRIES: ${MAX_POLLER_RETRIES:-3} - # Logging - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - volumes: - - logs-data:/app/logs - command: [ "node", "dist/processors/delayed/delayed.processor.js" ] - depends_on: - mongo: - condition: service_healthy - kafka: - condition: service_started - redis: - condition: service_healthy - loki: - condition: service_healthy - api: - condition: service_healthy - networks: - - ns-network - restart: unless-stopped - - # Recovery Service - Detects stuck notifications and creates alerts - recovery: - image: simplens-core:latest - env_file: - - .env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - # Recovery Configuration - RECOVERY_POLL_INTERVAL_MS: ${RECOVERY_POLL_INTERVAL_MS:-60000} - PROCESSING_STUCK_THRESHOLD_MS: ${PROCESSING_STUCK_THRESHOLD_MS:-300000} - PENDING_STUCK_THRESHOLD_MS: ${PENDING_STUCK_THRESHOLD_MS:-300000} - RECOVERY_BATCH_SIZE: ${RECOVERY_BATCH_SIZE:-50} - MAX_RETRY_COUNT: ${MAX_RETRY_COUNT:-5} - # Cleanup Configuration - CLEANUP_RESOLVED_ALERTS_RETENTION_MS: ${CLEANUP_RESOLVED_ALERTS_RETENTION_MS:-86400000} - CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS: ${CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS:-86400000} - # Logging - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - volumes: - - logs-data:/app/logs - command: [ "node", "dist/workers/recovery/recovery.service.js" ] - depends_on: - mongo: - condition: service_healthy - redis: - condition: service_healthy - loki: - condition: service_healthy - api: - condition: service_healthy - networks: - - ns-network - restart: unless-stopped - - # Admin Dashboard - Next.js web app for monitoring notifications - dashboard: - build: - context: ./dashboard - dockerfile: Dockerfile - container_name: dashboard - ports: - - "3002:3002" - env_file: - - .env - environment: - NODE_ENV: production - PORT: ${DASHBOARD_PORT:-3002} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - AUTH_SECRET: ${AUTH_SECRET:-change-this-in-production} - ADMIN_USERNAME: ${ADMIN_USERNAME:-admin} - ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin} - AUTH_TRUST_HOST: ${AUTH_TRUST_HOST:-true} - # For sending notifications - API_BASE_URL: http://api:${PORT:-3000} - NS_API_KEY: ${NS_API_KEY:-dev-api-key-change-in-production} - WEBHOOK_HOST: dashboard - WEBHOOK_PORT: "3002" - depends_on: - mongo: - condition: service_healthy - api: - condition: service_started - networks: - - ns-network - restart: unless-stopped - -volumes: - mongo_data: - mongo_config: - kafka_data: - redis_data: - loki_data: - grafana_data: - plugin-data: - logs-data: - - -networks: - ns-network: - driver: bridge diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 702edbc..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,95 +0,0 @@ -services: - api: - image: ghcr.io/simplenotificationsystem/simplens-core:latest - container_name: api - ports: - - 3000:3000 - env_file: - - .env - volumes: - - plugin-data:/app/.plugins - - logs-data:/app/logs - - ./simplens.config.yaml:/app/simplens.config.yaml:ro - command: [ "node", "dist/api/server.js" ] - networks: - - simplens - restart: unless-stopped - healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] - interval: 30s - timeout: 10s - retries: 3 - start_period: 10s - - worker: - image: ghcr.io/simplenotificationsystem/simplens-core:latest - env_file: - - .env - volumes: - - logs-data:/app/logs - command: [ "node", "dist/workers/worker.js" ] - networks: - - simplens - restart: unless-stopped - - notification_processor: - image: ghcr.io/simplenotificationsystem/simplens-core:latest - env_file: - - .env - volumes: - - plugin-data:/app/.plugins - - logs-data:/app/logs - - ./simplens.config.yaml:/app/simplens.config.yaml:ro - command: [ "node", "dist/processors/unified/unified.processor.js" ] - depends_on: - api: - condition: service_healthy - networks: - - simplens - restart: unless-stopped - - delayed_processor: - image: ghcr.io/simplenotificationsystem/simplens-core:latest - env_file: - - .env - volumes: - - logs-data:/app/logs - command: [ "node", "dist/processors/delayed/delayed.processor.js" ] - networks: - - simplens - restart: unless-stopped - - recovery: - image: ghcr.io/simplenotificationsystem/simplens-core:latest - env_file: - - .env - volumes: - - logs-data:/app/logs - command: [ "node", "dist/workers/recovery/recovery.service.js" ] - networks: - - simplens - restart: unless-stopped - - dashboard: - image: ghcr.io/simplenotificationsystem/simplens-dashboard:latest - ports: - - 3002:3002 - container_name: dashboard - env_file: - - .env - environment: - PORT: ${DASHBOARD_PORT:-3002} - API_BASE_URL: http://api:${PORT:-3000} - WEBHOOK_HOST: dashboard - WEBHOOK_PORT: ${DASHBOARD_PORT:-3002} - networks: - - simplens - restart: unless-stopped - -volumes: - plugin-data: - logs-data: - -networks: - simplens: - driver: bridge diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 0000000..696c3b8 --- /dev/null +++ b/docker-compose/README.md @@ -0,0 +1,123 @@ +# SimpleNS Multi-Environment Docker Compose Setup + +Run and test SimpleNS across three distinct environments (**Local**, **Development**, and **Master**) powered by a shared, persistent infrastructure stack. + +--- + +## 1. Quick Overview + +Instead of maintaining separate Kubernetes clusters or pods for routine branch testing, all environment switching is handled cleanly via Docker Compose: + +``` +docker-compose/ +├── docker-compose.infra.yaml # Shared base infrastructure (Mongo, Kafka, Redis, Loki, Grafana) +├── docker-compose.local.yaml # Application stack built from local source code +├── docker-compose.dev.yaml # Application stack using GHCR 'development' branch images +├── docker-compose.master.yaml # Application stack using GHCR 'latest' (master) release images +├── scripts/ +│ ├── switch-env.ps1 # Windows PowerShell switcher +│ └── switch-env.sh # Linux / macOS Bash switcher +└── README.md +``` + +### Key Highlights +- **Shared Infrastructure**: MongoDB replica set (`rs0`), Apache Kafka (KRaft), Redis, Loki, and Grafana run once and persist. +- **Zero Port Collisions**: All three environments use standard ports (`API: 3000`, `Dashboard: 3002`). The switching scripts cleanly terminate the previous environment before bringing up the new one. +- **Database Isolation**: + - `local` environment connects to `simplens_local` + - `dev` environment connects to `simplens_dev` + - `master` environment connects to `simplens_master` + +--- + +## 2. Usage & Switching Commands + +Use the unified switcher script from PowerShell (Windows) or Bash (Linux / macOS): + +### Windows (PowerShell) + +```powershell +# 1. Run Local source build (builds images from current working directory) +./docker-compose/scripts/switch-env.ps1 -Env local -Rebuild + +# 2. Run Development branch (pulls latest GHCR development image) +./docker-compose/scripts/switch-env.ps1 -Env dev -PullLatest + +# 3. Run Master branch (pulls latest GHCR master release) +./docker-compose/scripts/switch-env.ps1 -Env master -PullLatest + +# 4. Check active containers & service URLs +./docker-compose/scripts/switch-env.ps1 -Env status + +# 5. Stop application containers (keeps infrastructure running) +./docker-compose/scripts/switch-env.ps1 -Env down + +# 6. Stop everything (including infrastructure) +./docker-compose/scripts/switch-env.ps1 -Env infra-down +``` + +### Linux / macOS (Bash) + +```bash +# Make script executable +chmod +x ./docker-compose/scripts/switch-env.sh + +# 1. Run Local source build +./docker-compose/scripts/switch-env.sh local --rebuild + +# 2. Run Development branch +./docker-compose/scripts/switch-env.sh dev --pull + +# 3. Run Master branch +./docker-compose/scripts/switch-env.sh master --pull + +# 4. Check status +./docker-compose/scripts/switch-env.sh status + +# 5. Stop applications +./docker-compose/scripts/switch-env.sh down + +# 6. Stop everything +./docker-compose/scripts/switch-env.sh infra-down +``` + +--- + +## 3. Port & Service Directory + +When the stack is running, all services are accessible on your host machine: + +| Service | Host URL | Description / Credentials | +| :--- | :--- | :--- | +| **SimpleNS API** | [http://localhost:3000](http://localhost:3000) | Health endpoint: `/api/health` | +| **SimpleNS Dashboard** | [http://localhost:3002](http://localhost:3002) | Next.js admin management console | +| **Kafka UI** | [http://localhost:8081](http://localhost:8081) | Topic partitions, consumer groups, and lag | +| **Grafana** | [http://localhost:3001](http://localhost:3001) | User: `admin` / Password: `admin` | +| **MongoDB** | `mongodb://localhost:27017` | Direct database connection (`rs0`) | +| **Redis** | `redis://localhost:6379` | Direct Redis connection | + +--- + +## 4. Environment Comparison + +| Attribute | `local` | `dev` | `master` | +| :--- | :--- | :--- | :--- | +| **Image Source** | Local Dockerfile build | `ghcr.io/.../simplens-core:development` | `ghcr.io/.../simplens-core:latest` | +| **Database** | `simplens_local` | `simplens_dev` | `simplens_master` | +| **Use Case** | Active code development & testing | Testing pre-release features | Testing production-equivalent baseline | +| **Rebuild Flag** | `-Rebuild` / `--rebuild` | N/A | N/A | +| **Pull Flag** | N/A | `-PullLatest` / `--pull` | `-PullLatest` / `--pull` | + +--- + +## 5. Troubleshooting & FAQ + +### Port already in use (e.g. 3000 or 3002)? +Run `./docker-compose/scripts/switch-env.ps1 -Env down` to shut down any lingering application containers. If an old root Docker Compose is running, stop it via `docker compose down`. + +### Resetting all data? +To completely reset all databases and queues: +```powershell +./docker-compose/scripts/switch-env.ps1 -Env infra-down +docker volume prune -f +``` diff --git a/docker-compose/docker-compose.dev.yaml b/docker-compose/docker-compose.dev.yaml new file mode 100644 index 0000000..4bf71f5 --- /dev/null +++ b/docker-compose/docker-compose.dev.yaml @@ -0,0 +1,190 @@ +# ============================================================================== +# SimpleNS Development Environment (GHCR 'development' branch images) +# Connects to shared 'simplens-infra' network and 'simplens_dev' database. +# ============================================================================== + +name: simplens-dev + +services: + # ============================================ + # 1. API Server + # ============================================ + api: + image: ghcr.io/simplenotificationsystem/simplens-core:development + container_name: simplens-api-dev + ports: + - "3000:3000" + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-development} + PORT: ${PORT:-3000} + MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} + volumes: + - logs-data-dev:/app/logs + - ../simplens.config.yaml:/app/simplens.config.yaml:ro + command: [ "node", "dist/api/server.js" ] + networks: + - simplens-infra + restart: unless-stopped + healthcheck: + test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] + interval: 15s + timeout: 5s + retries: 3 + start_period: 10s + labels: + simplens.environment: "dev" + + # ============================================ + # 2. Outbox Worker + # ============================================ + worker: + image: ghcr.io/simplenotificationsystem/simplens-core:development + container_name: simplens-worker-dev + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-development} + MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-dev:/app/logs + command: [ "node", "dist/workers/worker.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "dev" + + # ============================================ + # 3. Unified Notification Processor + # ============================================ + notification_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:development + container_name: simplens-notification-processor-dev + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-development} + MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + PROCESSOR_CHANNEL: ${PROCESSOR_CHANNEL:-all} + SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-dev:/app/logs + - ../simplens.config.yaml:/app/simplens.config.yaml:ro + command: [ "node", "dist/processors/unified/unified.processor.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "dev" + + # ============================================ + # 4. Delayed Processor (Scheduled Notifications) + # ============================================ + delayed_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:development + container_name: simplens-delayed-processor-dev + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-development} + MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-dev:/app/logs + command: [ "node", "dist/processors/delayed/delayed.processor.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "dev" + + # ============================================ + # 5. Recovery Service (Stuck Notifications & Audits) + # ============================================ + recovery: + image: ghcr.io/simplenotificationsystem/simplens-core:development + container_name: simplens-recovery-dev + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-development} + MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-dev:/app/logs + command: [ "node", "dist/workers/recovery/recovery.service.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "dev" + + # ============================================ + # 6. Admin Dashboard (Next.js) + # ============================================ + dashboard: + image: ghcr.io/simplenotificationsystem/simplens-dashboard:development + container_name: simplens-dashboard-dev + ports: + - "3002:3002" + env_file: + - ../.env + environment: + NODE_ENV: production + PORT: ${DASHBOARD_PORT:-3002} + API_BASE_URL: http://api:${PORT:-3000} + MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 + AUTH_SECRET: ${AUTH_SECRET:-change-this-in-production} + ADMIN_USERNAME: ${ADMIN_USERNAME:-admin} + ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin} + AUTH_TRUST_HOST: "true" + WEBHOOK_HOST: dashboard + WEBHOOK_PORT: "3002" + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "dev" + +volumes: + logs-data-dev: + +networks: + simplens-infra: + external: true + name: simplens-infra diff --git a/docker-compose.infra.yaml b/docker-compose/docker-compose.infra.yaml similarity index 58% rename from docker-compose.infra.yaml rename to docker-compose/docker-compose.infra.yaml index 1a3cb16..86553c2 100644 --- a/docker-compose.infra.yaml +++ b/docker-compose/docker-compose.infra.yaml @@ -1,19 +1,21 @@ -# ============================================ -# INFRA_HOST: Set this in .env to your machine's IP/hostname when running -# infrastructure on a separate system from application services. -# Default: host.docker.internal (for same-system deployment) -# ============================================ +# ============================================================================== +# SimpleNS Shared Infrastructure Services +# Runs Mongo (Replica Set), Kafka (KRaft), Kafka-UI, Redis, Loki, Grafana +# Shared across all application environments (local, dev, master). +# ============================================================================== + +name: infra services: # ============================================ - # Infrastructure Services + # 1. MongoDB (Replica Set rs0 for Transactions) # ============================================ mongo: image: mongo:7.0 - container_name: mongo + container_name: simplens-infra-mongo command: [ "--replSet", "rs0", "--bind_ip_all", "--port", "27017" ] ports: - - 27017:27017 + - "27017:27017" extra_hosts: - "host.docker.internal:host-gateway" healthcheck: @@ -24,84 +26,98 @@ services: start_interval: 1s retries: 30 volumes: - - "mongo_data:/data/db" - - "mongo_config:/data/configdb" + - mongo_data:/data/db + - mongo_config:/data/configdb + networks: + - simplens-infra + restart: unless-stopped + # ============================================ + # 2. Apache Kafka (KRaft Native Mode) + # ============================================ kafka: image: apache/kafka-native - container_name: kafka + container_name: simplens-infra-kafka ports: - "9092:9092" environment: - # Configure listeners for both docker and host communication KAFKA_LISTENERS: CONTROLLER://localhost:9091,HOST://0.0.0.0:9092,DOCKER://0.0.0.0:9093 KAFKA_ADVERTISED_LISTENERS: HOST://${INFRA_HOST:-host.docker.internal}:9092,DOCKER://kafka:9093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,DOCKER:PLAINTEXT,HOST:PLAINTEXT - - # Settings required for KRaft mode KAFKA_NODE_ID: 1 KAFKA_PROCESS_ROLES: broker,controller KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9091 - - # Listener to use for broker-to-broker communication KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER - - # Required for a single node cluster KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - - # Disable auto-topic creation - API server will create topics with correct partitions KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" volumes: - - "kafka_data:/var/lib/kafka/data" + - kafka_data:/var/lib/kafka/data + networks: + - simplens-infra + restart: unless-stopped + # ============================================ + # 3. Kafka UI (Topic & Consumer Explorer) + # ============================================ kafka-ui: image: kafbat/kafka-ui:main - container_name: kafka-ui + container_name: simplens-infra-kafka-ui ports: - - 8080:8080 + - "8081:8080" environment: DYNAMIC_CONFIG_ENABLED: "true" KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9093 depends_on: - kafka + networks: + - simplens-infra + restart: unless-stopped + # ============================================ + # 4. Redis (Idempotency, Queue, Pub/Sub) + # ============================================ redis: image: redis:7-alpine - container_name: redis + container_name: simplens-infra-redis ports: - "6379:6379" command: redis-server --appendonly yes volumes: - - "redis_data:/data" + - redis_data:/data + networks: + - simplens-infra healthcheck: test: [ "CMD", "redis-cli", "ping" ] interval: 5s timeout: 3s retries: 5 + restart: unless-stopped # ============================================ - # Observability Services + # 5. Observability (Loki & Grafana) # ============================================ - loki: image: grafana/loki:2.9.0 - container_name: loki + container_name: simplens-infra-loki ports: - "3100:3100" command: -config.file=/etc/loki/local-config.yaml volumes: - - "loki_data:/loki" + - loki_data:/loki + networks: + - simplens-infra healthcheck: test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1" ] interval: 10s timeout: 5s retries: 5 + restart: unless-stopped grafana: image: grafana/grafana:10.2.0 - container_name: grafana + container_name: simplens-infra-grafana ports: - "3001:3000" environment: @@ -110,10 +126,13 @@ services: - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - GF_SECURITY_ADMIN_PASSWORD=admin volumes: - - "grafana_data:/var/lib/grafana" + - grafana_data:/var/lib/grafana depends_on: loki: condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped volumes: mongo_data: @@ -121,4 +140,9 @@ volumes: kafka_data: redis_data: loki_data: - grafana_data: \ No newline at end of file + grafana_data: + +networks: + simplens-infra: + name: simplens-infra + driver: bridge diff --git a/docker-compose/docker-compose.local.yaml b/docker-compose/docker-compose.local.yaml new file mode 100644 index 0000000..f61d676 --- /dev/null +++ b/docker-compose/docker-compose.local.yaml @@ -0,0 +1,196 @@ +# ============================================================================== +# SimpleNS Local Environment (Built from Local Source Code) +# Connects to shared 'simplens-infra' network and 'simplens_local' database. +# ============================================================================== + +name: simplens-local + +services: + # ============================================ + # 1. API Server + # ============================================ + api: + build: + context: .. + dockerfile: Dockerfile + image: simplens-core:local + container_name: simplens-api-local + ports: + - "3000:3000" + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-development} + PORT: ${PORT:-3000} + MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} + volumes: + - logs-data-local:/app/logs + - ../simplens.config.yaml:/app/simplens.config.yaml:ro + command: [ "node", "dist/api/server.js" ] + networks: + - simplens-infra + restart: unless-stopped + healthcheck: + test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] + interval: 15s + timeout: 5s + retries: 3 + start_period: 10s + labels: + simplens.environment: "local" + + # ============================================ + # 2. Outbox Worker + # ============================================ + worker: + image: simplens-core:local + container_name: simplens-worker-local + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-development} + MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-local:/app/logs + command: [ "node", "dist/workers/worker.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "local" + + # ============================================ + # 3. Unified Notification Processor + # ============================================ + notification_processor: + image: simplens-core:local + container_name: simplens-notification-processor-local + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-development} + MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + PROCESSOR_CHANNEL: ${PROCESSOR_CHANNEL:-all} + SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-local:/app/logs + - ../simplens.config.yaml:/app/simplens.config.yaml:ro + command: [ "node", "dist/processors/unified/unified.processor.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "local" + + # ============================================ + # 4. Delayed Processor (Scheduled Notifications) + # ============================================ + delayed_processor: + image: simplens-core:local + container_name: simplens-delayed-processor-local + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-development} + MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-local:/app/logs + command: [ "node", "dist/processors/delayed/delayed.processor.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "local" + + # ============================================ + # 5. Recovery Service (Stuck Notifications & Audits) + # ============================================ + recovery: + image: simplens-core:local + container_name: simplens-recovery-local + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-development} + MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-local:/app/logs + command: [ "node", "dist/workers/recovery/recovery.service.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "local" + + # ============================================ + # 6. Admin Dashboard (Next.js) + # ============================================ + dashboard: + build: + context: ../dashboard + dockerfile: Dockerfile + image: simplens-dashboard:local + container_name: simplens-dashboard-local + ports: + - "3002:3002" + env_file: + - ../.env + environment: + NODE_ENV: production + PORT: ${DASHBOARD_PORT:-3002} + API_BASE_URL: http://api:${PORT:-3000} + MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 + AUTH_SECRET: ${AUTH_SECRET:-change-this-in-production} + ADMIN_USERNAME: ${ADMIN_USERNAME:-admin} + ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin} + AUTH_TRUST_HOST: "true" + WEBHOOK_HOST: dashboard + WEBHOOK_PORT: "3002" + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "local" + +volumes: + logs-data-local: + +networks: + simplens-infra: + external: true + name: simplens-infra diff --git a/docker-compose/docker-compose.master.yaml b/docker-compose/docker-compose.master.yaml new file mode 100644 index 0000000..7643cc4 --- /dev/null +++ b/docker-compose/docker-compose.master.yaml @@ -0,0 +1,190 @@ +# ============================================================================== +# SimpleNS Master Environment (GHCR 'latest' / master branch images) +# Connects to shared 'simplens-infra' network and 'simplens_master' database. +# ============================================================================== + +name: simplens-master + +services: + # ============================================ + # 1. API Server + # ============================================ + api: + image: ghcr.io/simplenotificationsystem/simplens-core:latest + container_name: simplens-api-master + ports: + - "3000:3000" + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-production} + PORT: ${PORT:-3000} + MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} + volumes: + - logs-data-master:/app/logs + - ../simplens.config.yaml:/app/simplens.config.yaml:ro + command: [ "node", "dist/api/server.js" ] + networks: + - simplens-infra + restart: unless-stopped + healthcheck: + test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] + interval: 15s + timeout: 5s + retries: 3 + start_period: 10s + labels: + simplens.environment: "master" + + # ============================================ + # 2. Outbox Worker + # ============================================ + worker: + image: ghcr.io/simplenotificationsystem/simplens-core:latest + container_name: simplens-worker-master + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-production} + MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-master:/app/logs + command: [ "node", "dist/workers/worker.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "master" + + # ============================================ + # 3. Unified Notification Processor + # ============================================ + notification_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:latest + container_name: simplens-notification-processor-master + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-production} + MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + PROCESSOR_CHANNEL: ${PROCESSOR_CHANNEL:-all} + SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-master:/app/logs + - ../simplens.config.yaml:/app/simplens.config.yaml:ro + command: [ "node", "dist/processors/unified/unified.processor.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "master" + + # ============================================ + # 4. Delayed Processor (Scheduled Notifications) + # ============================================ + delayed_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:latest + container_name: simplens-delayed-processor-master + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-production} + MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + BROKERS: kafka:9093 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-master:/app/logs + command: [ "node", "dist/processors/delayed/delayed.processor.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "master" + + # ============================================ + # 5. Recovery Service (Stuck Notifications & Audits) + # ============================================ + recovery: + image: ghcr.io/simplenotificationsystem/simplens-core:latest + container_name: simplens-recovery-master + env_file: + - ../.env + environment: + NODE_ENV: ${NODE_ENV:-production} + MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 + REDIS_URL: redis://redis:6379 + LOKI_URL: http://loki:3100 + LOG_LEVEL: ${LOG_LEVEL:-info} + volumes: + - logs-data-master:/app/logs + command: [ "node", "dist/workers/recovery/recovery.service.js" ] + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "master" + + # ============================================ + # 6. Admin Dashboard (Next.js) + # ============================================ + dashboard: + image: ghcr.io/simplenotificationsystem/simplens-dashboard:latest + container_name: simplens-dashboard-master + ports: + - "3002:3002" + env_file: + - ../.env + environment: + NODE_ENV: production + PORT: ${DASHBOARD_PORT:-3002} + API_BASE_URL: http://api:${PORT:-3000} + MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 + AUTH_SECRET: ${AUTH_SECRET:-change-this-in-production} + ADMIN_USERNAME: ${ADMIN_USERNAME:-admin} + ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin} + AUTH_TRUST_HOST: "true" + WEBHOOK_HOST: dashboard + WEBHOOK_PORT: "3002" + depends_on: + api: + condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped + labels: + simplens.environment: "master" + +volumes: + logs-data-master: + +networks: + simplens-infra: + external: true + name: simplens-infra diff --git a/docker-compose/scripts/switch-env.ps1 b/docker-compose/scripts/switch-env.ps1 new file mode 100644 index 0000000..433c7d6 --- /dev/null +++ b/docker-compose/scripts/switch-env.ps1 @@ -0,0 +1,217 @@ +<# +.SYNOPSIS + Switches between SimpleNS Docker Compose environments (local, dev, master). + Ensures shared infrastructure is running while preventing container/port collisions. + +.PARAMETER Env + Environment to run: 'local', 'dev', 'master', 'status', 'down', 'infra-only', 'infra-down'. + Default: 'status' + +.PARAMETER Rebuild + (Local only) Rebuilds the local Docker images from source code before starting. + +.PARAMETER NoCache + (Local only) Rebuilds local Docker images without using Docker cache. + +.PARAMETER PullLatest + (Dev/Master only) Pulls the latest images from GHCR before starting. +#> + +param( + [ValidateSet("local", "dev", "master", "status", "down", "infra-only", "infra-down")] + [string]$Env = "status", + + [switch]$Rebuild, + [switch]$NoCache, + [switch]$PullLatest +) + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$ComposeDir = Split-Path -Parent $ScriptDir +$ProjectRoot = Split-Path -Parent $ComposeDir + +$InfraCompose = Join-Path $ComposeDir "docker-compose.infra.yaml" +$LocalCompose = Join-Path $ComposeDir "docker-compose.local.yaml" +$DevCompose = Join-Path $ComposeDir "docker-compose.dev.yaml" +$MasterCompose = Join-Path $ComposeDir "docker-compose.master.yaml" + +$EnvFile = Join-Path $ProjectRoot ".env" +if (-not (Test-Path $EnvFile)) { + $EnvFile = Join-Path $ProjectRoot ".env.example" +} + +function Ensure-Network { + $networkExists = docker network ls --filter "name=^simplens-infra$" --format "{{.Name}}" + if (-not $networkExists) { + Write-Host "[Network] Creating 'simplens-infra' bridge network..." -ForegroundColor Cyan + docker network create simplens-infra | Out-Null + } +} + +function Ensure-InfraImages { + Write-Host "[Infra] Verifying infrastructure Docker images..." -ForegroundColor Cyan + $images = @() + try { + $rawImages = docker compose -p infra --env-file $EnvFile -f $InfraCompose config --images 2>$null + if ($rawImages) { + $images = ($rawImages -split "`r?`n") | Where-Object { $_.Trim() -ne "" } | ForEach-Object { $_.Trim() } + } + } catch {} + + if (-not $images -or $images.Count -eq 0) { + $images = @( + "mongo:7.0", + "apache/kafka-native", + "kafbat/kafka-ui:main", + "redis:7-alpine", + "grafana/loki:2.9.0", + "grafana/grafana:10.2.0" + ) + } + + $missingImages = @() + foreach ($img in $images) { + $found = docker images -q $img 2>$null + if (-not $found) { + $missingImages += $img + } + } + + if ($missingImages.Count -gt 0) { + Write-Host "[Infra] Found $($missingImages.Count) missing infrastructure image(s). Pulling..." -ForegroundColor Yellow + foreach ($img in $missingImages) { + Write-Host "[Pull] Pulling '$img'..." -ForegroundColor Cyan + docker pull $img + } + Write-Host "[Infra] All required infrastructure images are downloaded." -ForegroundColor Green + } else { + Write-Host "[Infra] All infrastructure images are present locally." -ForegroundColor Green + } +} + +function Start-Infra { + Ensure-Network + Ensure-InfraImages + Write-Host "`n[Infra] Starting shared infrastructure services..." -ForegroundColor Cyan + docker compose -p infra --env-file $EnvFile -f $InfraCompose up -d + Write-Host "[Infra] Infrastructure services are up and healthy." -ForegroundColor Green +} + +function Stop-AppEnvironments { + param([string]$Except = "") + + if ($Except -ne "local") { + $localRunning = docker ps --filter "label=simplens.environment=local" --format "{{.ID}}" + if ($localRunning) { + Write-Host "[Stop] Stopping 'local' environment..." -ForegroundColor Yellow + docker compose -p simplens-local --env-file $EnvFile -f $LocalCompose down + } + } + if ($Except -ne "dev") { + $devRunning = docker ps --filter "label=simplens.environment=dev" --format "{{.ID}}" + if ($devRunning) { + Write-Host "[Stop] Stopping 'dev' environment..." -ForegroundColor Yellow + docker compose -p simplens-dev --env-file $EnvFile -f $DevCompose down + } + } + if ($Except -ne "master") { + $masterRunning = docker ps --filter "label=simplens.environment=master" --format "{{.ID}}" + if ($masterRunning) { + Write-Host "[Stop] Stopping 'master' environment..." -ForegroundColor Yellow + docker compose -p simplens-master --env-file $EnvFile -f $MasterCompose down + } + } +} + +function Show-Status { + Write-Host "`n============================================================" -ForegroundColor Cyan + Write-Host " SimpleNS Container Status " -ForegroundColor Cyan + Write-Host "============================================================" -ForegroundColor Cyan + + Write-Host "`n--- Infrastructure Containers (shared: infra) ---" -ForegroundColor Yellow + docker ps --filter "network=simplens-infra" --filter "name=simplens-infra" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" + + Write-Host "`n--- Active Application Containers ---" -ForegroundColor Yellow + docker ps --filter "network=simplens-infra" --filter "label=simplens.environment" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" + + Write-Host "`n--- Service URLs ---" -ForegroundColor Green + Write-Host " API Server: http://localhost:3000/api/health" -ForegroundColor White + Write-Host " Dashboard: http://localhost:3002" -ForegroundColor White + Write-Host " Kafka UI: http://localhost:8081" -ForegroundColor White + Write-Host " Grafana: http://localhost:3001 (User: admin / Pass: admin)" -ForegroundColor White + Write-Host "============================================================`n" -ForegroundColor Cyan +} + +switch ($Env) { + "local" { + Start-Infra + Stop-AppEnvironments -Except "local" + + if ($Rebuild -or $NoCache) { + Write-Host "`n[Build] Building local Docker images from source..." -ForegroundColor Cyan + $buildArgs = @("compose", "-p", "simplens-local", "--env-file", $EnvFile, "-f", $LocalCompose, "build") + if ($NoCache) { $buildArgs += "--no-cache" } + docker @buildArgs + } + + Write-Host "`n[Start] Starting 'local' environment..." -ForegroundColor Cyan + docker compose -p simplens-local --env-file $EnvFile -f $LocalCompose up -d + Write-Host "[OK] 'Local' environment is running!" -ForegroundColor Green + Show-Status + } + + "dev" { + Start-Infra + Stop-AppEnvironments -Except "dev" + + if ($PullLatest) { + Write-Host "`n[Pull] Pulling latest development images from GHCR..." -ForegroundColor Cyan + docker compose -p simplens-dev --env-file $EnvFile -f $DevCompose pull + } + + Write-Host "`n[Start] Starting 'dev' environment (GHCR development branch)..." -ForegroundColor Cyan + docker compose -p simplens-dev --env-file $EnvFile -f $DevCompose up -d + Write-Host "[OK] 'Dev' environment is running!" -ForegroundColor Green + Show-Status + } + + "master" { + Start-Infra + Stop-AppEnvironments -Except "master" + + if ($PullLatest) { + Write-Host "`n[Pull] Pulling latest master images from GHCR..." -ForegroundColor Cyan + docker compose -p simplens-master --env-file $EnvFile -f $MasterCompose pull + } + + Write-Host "`n[Start] Starting 'master' environment (GHCR latest release)..." -ForegroundColor Cyan + docker compose -p simplens-master --env-file $EnvFile -f $MasterCompose up -d + Write-Host "[OK] 'Master' environment is running!" -ForegroundColor Green + Show-Status + } + + "status" { + Show-Status + } + + "down" { + Write-Host "`n[Down] Stopping all active application environments..." -ForegroundColor Yellow + Stop-AppEnvironments + Write-Host "[OK] All application containers stopped. Infrastructure remains active." -ForegroundColor Green + } + + "infra-only" { + Stop-AppEnvironments + Start-Infra + Write-Host "[OK] Infrastructure services are running." -ForegroundColor Green + } + + "infra-down" { + Write-Host "`n[Infra-Down] Stopping all applications and shared infrastructure..." -ForegroundColor Yellow + Stop-AppEnvironments + docker compose -p infra --env-file $EnvFile -f $InfraCompose down + Write-Host "[OK] All SimpleNS containers stopped." -ForegroundColor Green + } +} diff --git a/docker-compose/scripts/switch-env.sh b/docker-compose/scripts/switch-env.sh new file mode 100644 index 0000000..51845d8 --- /dev/null +++ b/docker-compose/scripts/switch-env.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# ============================================================================== +# SimpleNS Docker Compose Environment Switcher (Linux / macOS / WSL) +# Switches between 'local', 'dev', and 'master' environments. +# ============================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +PROJECT_ROOT="$(cd "$COMPOSE_DIR/.." && pwd)" + +INFRA_COMPOSE="$COMPOSE_DIR/docker-compose.infra.yaml" +LOCAL_COMPOSE="$COMPOSE_DIR/docker-compose.local.yaml" +DEV_COMPOSE="$COMPOSE_DIR/docker-compose.dev.yaml" +MASTER_COMPOSE="$COMPOSE_DIR/docker-compose.master.yaml" + +ENV_FILE="$PROJECT_ROOT/.env" +if [ ! -f "$ENV_FILE" ]; then + ENV_FILE="$PROJECT_ROOT/.env.example" +fi + +ENV="${1:-status}" +REBUILD=false +NOCACHE=false +PULL_LATEST=false + +# Parse flags +shift || true +while [ $# -gt 0 ]; do + case "$1" in + --rebuild|-r) REBUILD=true ;; + --no-cache) NOCACHE=true; REBUILD=true ;; + --pull|-p) PULL_LATEST=true ;; + *) echo "Unknown flag: $1" >&2; exit 1 ;; + esac + shift +done + +ensure_network() { + if ! docker network inspect simplens-infra >/dev/null 2>&1; then + echo "[Network] Creating 'simplens-infra' bridge network..." + docker network create simplens-infra >/dev/null 2>&1 || true + fi +} + +ensure_infra_images() { + echo "[Infra] Verifying infrastructure Docker images..." + local images=( + "mongo:7.0" + "apache/kafka-native" + "kafbat/kafka-ui:main" + "redis:7-alpine" + "grafana/loki:2.9.0" + "grafana/grafana:10.2.0" + ) + + local missing_images=() + for img in "${images[@]}"; do + if ! docker image inspect "$img" >/dev/null 2>&1; then + missing_images+=("$img") + fi + done + + if [ ${#missing_images[@]} -gt 0 ]; then + echo "[Infra] Found ${#missing_images[@]} missing infrastructure image(s). Pulling..." + for img in "${missing_images[@]}"; do + echo "[Pull] Pulling '$img'..." + docker pull "$img" || true + done + echo "[Infra] All required infrastructure images are downloaded." + else + echo "[Infra] All infrastructure images are present locally." + fi +} + +start_infra() { + ensure_network + ensure_infra_images + echo "[Infra] Starting shared infrastructure services..." + docker compose -p infra --env-file "$ENV_FILE" -f "$INFRA_COMPOSE" up -d + echo "[Infra] Infrastructure services are up and healthy." +} + +stop_app_environments() { + local except="${1:-}" + + if [ "$except" != "local" ]; then + local local_running + local_running=$(docker ps -q --filter "label=simplens.environment=local" 2>/dev/null || true) + if [ -n "$local_running" ]; then + echo "[Stop] Stopping 'local' environment..." + docker compose -p simplens-local --env-file "$ENV_FILE" -f "$LOCAL_COMPOSE" down 2>/dev/null || true + fi + fi + + if [ "$except" != "dev" ]; then + local dev_running + dev_running=$(docker ps -q --filter "label=simplens.environment=dev" 2>/dev/null || true) + if [ -n "$dev_running" ]; then + echo "[Stop] Stopping 'dev' environment..." + docker compose -p simplens-dev --env-file "$ENV_FILE" -f "$DEV_COMPOSE" down 2>/dev/null || true + fi + fi + + if [ "$except" != "master" ]; then + local master_running + master_running=$(docker ps -q --filter "label=simplens.environment=master" 2>/dev/null || true) + if [ -n "$master_running" ]; then + echo "[Stop] Stopping 'master' environment..." + docker compose -p simplens-master --env-file "$ENV_FILE" -f "$MASTER_COMPOSE" down 2>/dev/null || true + fi + fi +} + +show_status() { + echo "" + echo "============================================================" + echo " SimpleNS Container Status " + echo "============================================================" + echo "" + echo "--- Infrastructure Containers (shared: infra) ---" + docker ps --filter "network=simplens-infra" --filter "name=simplens-infra" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" + echo "" + echo "--- Active Application Containers ---" + docker ps --filter "network=simplens-infra" --filter "label=simplens.environment" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" + echo "" + echo "--- Service URLs ---" + echo " API Server: http://localhost:3000/api/health" + echo " Dashboard: http://localhost:3002" + echo " Kafka UI: http://localhost:8081" + echo " Grafana: http://localhost:3001 (User: admin / Pass: admin)" + echo "============================================================" + echo "" +} + +case "$ENV" in + local) + start_infra + stop_app_environments "local" + if [ "$REBUILD" = true ]; then + echo "[Build] Building local Docker images from source..." + if [ "$NOCACHE" = true ]; then + docker compose -p simplens-local --env-file "$ENV_FILE" -f "$LOCAL_COMPOSE" build --no-cache + else + docker compose -p simplens-local --env-file "$ENV_FILE" -f "$LOCAL_COMPOSE" build + fi + fi + echo "[Start] Starting 'local' environment..." + docker compose -p simplens-local --env-file "$ENV_FILE" -f "$LOCAL_COMPOSE" up -d + echo "[OK] 'Local' environment is running!" + show_status + ;; + + dev) + start_infra + stop_app_environments "dev" + if [ "$PULL_LATEST" = true ]; then + echo "[Pull] Pulling latest development images from GHCR..." + docker compose -p simplens-dev --env-file "$ENV_FILE" -f "$DEV_COMPOSE" pull + fi + echo "[Start] Starting 'dev' environment..." + docker compose -p simplens-dev --env-file "$ENV_FILE" -f "$DEV_COMPOSE" up -d + echo "[OK] 'Dev' environment is running!" + show_status + ;; + + master) + start_infra + stop_app_environments "master" + if [ "$PULL_LATEST" = true ]; then + echo "[Pull] Pulling latest master images from GHCR..." + docker compose -p simplens-master --env-file "$ENV_FILE" -f "$MASTER_COMPOSE" pull + fi + echo "[Start] Starting 'master' environment..." + docker compose -p simplens-master --env-file "$ENV_FILE" -f "$MASTER_COMPOSE" up -d + echo "[OK] 'Master' environment is running!" + show_status + ;; + + status) + show_status + ;; + + down) + echo "[Down] Stopping all active application environments..." + stop_app_environments "" + echo "[OK] All application containers stopped. Infrastructure remains active." + ;; + + infra-only) + stop_app_environments "" + start_infra + echo "[OK] Infrastructure services are running." + ;; + + infra-down) + echo "[Infra-Down] Stopping all applications and shared infrastructure..." + stop_app_environments "" + docker compose -p infra --env-file "$ENV_FILE" -f "$INFRA_COMPOSE" down + echo "[OK] All SimpleNS containers stopped." + ;; + + *) + echo "Usage: $0 {local|dev|master|status|down|infra-only|infra-down} [--rebuild] [--no-cache] [--pull]" + exit 1 + ;; +esac From 3e58502150c989af67f293216f1fecbb807ad093 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Mon, 14 Sep 2026 03:00:30 +0530 Subject: [PATCH 02/46] Refactor testing scripts: remove chaos testing scripts and update README --- .devcontainer/devcontainer.json | 2 +- k8s/README.md | 197 ----- k8s/apps/app-development.yaml | 236 ------ k8s/apps/app-local.yaml | 236 ------ k8s/apps/app-master.yaml | 236 ------ k8s/base/configmap-simplens.yaml | 58 -- k8s/base/namespace.yaml | 6 - k8s/infra/pod-infra.yaml | 201 ----- k8s/infra/pvc-infra.yaml | 71 -- k8s/infra/services-infra.yaml | 110 --- k8s/kind-config.yaml | 58 -- k8s/scripts/win/build-local.ps1 | 92 --- k8s/scripts/win/deploy-all.ps1 | 77 -- k8s/scripts/win/port-forward.ps1 | 154 ---- k8s/scripts/win/switch-env.ps1 | 155 ---- scripts/README.md | 98 +-- scripts/crash-tests/crash-multiwave-all.sh | 843 --------------------- scripts/crash-tests/k8s-chaos.ps1 | 159 ---- 18 files changed, 3 insertions(+), 2986 deletions(-) delete mode 100644 k8s/README.md delete mode 100644 k8s/apps/app-development.yaml delete mode 100644 k8s/apps/app-local.yaml delete mode 100644 k8s/apps/app-master.yaml delete mode 100644 k8s/base/configmap-simplens.yaml delete mode 100644 k8s/base/namespace.yaml delete mode 100644 k8s/infra/pod-infra.yaml delete mode 100644 k8s/infra/pvc-infra.yaml delete mode 100644 k8s/infra/services-infra.yaml delete mode 100644 k8s/kind-config.yaml delete mode 100644 k8s/scripts/win/build-local.ps1 delete mode 100644 k8s/scripts/win/deploy-all.ps1 delete mode 100644 k8s/scripts/win/port-forward.ps1 delete mode 100644 k8s/scripts/win/switch-env.ps1 delete mode 100644 scripts/crash-tests/crash-multiwave-all.sh delete mode 100644 scripts/crash-tests/k8s-chaos.ps1 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 62fd3e5..069928c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -19,7 +19,7 @@ } }, "postCreateCommand": "npm install && cd dashboard && npm install", - "postStartCommand": "docker compose -f docker-compose.infra.yaml up -d", + "postStartCommand": "docker compose -f ./docker-compose/docker-compose.infra.yaml up -d", "remoteEnv": { "NODE_ENV": "development", "INFRA_HOST": "localhost" diff --git a/k8s/README.md b/k8s/README.md deleted file mode 100644 index 1c61a90..0000000 --- a/k8s/README.md +++ /dev/null @@ -1,197 +0,0 @@ -# SimpleNS Kubernetes Setup Guide - -Run SimpleNS in a local Kubernetes cluster (Docker Desktop or Kind) with testing environments for **Local builds**, **Master branch (Prod)**, and **Development branch**, all sharing a single infrastructure pod. - ---- - -## 1. Quick Start in 4 Steps - -### Step 0: Create the Kind Cluster (One-time Setup) -Ensure you have `kind` installed (`winget install Kubernetes.kind` or `choco install kind`), then create the cluster: -```powershell -kind create cluster --name simplens --config k8s/kind-config.yaml -``` - ---- - -### Step 1: Build Local Images -Build the Docker images for your local code: -```powershell -./k8s/scripts/win/build-local.ps1 -``` -> **Note:** The script automatically detects the running Kind cluster (from your kubectl context or Kind cluster list), loads `simplens-core:local` and `simplens-dashboard:local` into Kind, and prunes dangling `:` images to prevent duplicate image clutter in Docker Desktop. -> If you need a clean rebuild without Docker cache, pass `-NoCache`: -> ```powershell -> ./k8s/scripts/win/build-local.ps1 -NoCache -> ``` - ---- - -### Step 2: Deploy to Kubernetes -Deploy the shared infrastructure pod and start the local environment: -```powershell -./k8s/scripts/win/deploy-all.ps1 -ActiveEnv local -``` -This script automatically: -- Creates the `simplens` namespace -- Injects your `.env` file as a Kubernetes Secret (`app-env`) -- Provisions persistent storage for Mongo, Kafka, Redis, and logs -- Starts the **Infra Pod** (Mongo, Kafka, Kafka-UI, Redis, Loki, Grafana) -- Starts the **Local App Pod** (API, Worker, Processor, Delayed Processor, Recovery, Dashboard) - ---- - -### Step 3: Open Port Forwarding (Access in Browser) -In a separate terminal tab, start the port forwarder: -```powershell -./k8s/scripts/win/port-forward.ps1 -``` -*(The script automatically cleans up previous port-forward processes and avoids port conflicts with Kind NodePorts).* - -Now open your browser: - -| Service | Browser URL | Credentials / Notes | -| :--- | :--- | :--- | -| **SimpleNS Dashboard** | [http://localhost:3002](http://localhost:3002) *(or NodePort :30302)* | Admin web interface | -| **API Health Check** | [http://localhost:3000/api/health](http://localhost:3000/api/health) *(or NodePort :30300)* | Returns `{"status":"ok"}` | -| **Kafka UI** | [http://localhost:8080](http://localhost:8080) *(or NodePort :30080)* | Topic & message explorer | -| **Grafana** | [http://localhost:3001](http://localhost:3001) *(or NodePort :30001)* | User: `admin` / Password: `admin` | - -*(Keep this terminal open while testing. Press `Ctrl + C` to stop port forwarding).* - ---- - -## 2. Iterating on Local Code (Hot Rebuilding) - -Whenever you make code changes and want to update the running local pod: -```powershell -./k8s/scripts/win/switch-env.ps1 -Env local -PullLatest -``` -This single command will: -1. Rebuild the local Docker images from your current source code. -2. Load the updated images into your Kind cluster. -3. Clean up dangling images so duplicates don't accumulate. -4. Trigger a rolling restart of `app-local` and wait for the new pod rollout to complete before returning. - -> **Tip:** If updating Dashboard UI components, do a hard refresh in the browser (`Ctrl + Shift + R` or `Ctrl + F5`) to bypass cached Next.js static bundles. - ---- - -## 3. Helper Scripts Reference - -All scripts live in `k8s/scripts/win/` and are designed to be simple: - -### `build-local.ps1` -Builds `simplens-core:local` and `simplens-dashboard:local` from your current source code, loads them into Kind, and prunes dangling images. -```powershell -# Standard build: -./k8s/scripts/win/build-local.ps1 - -# Clean build without Docker cache: -./k8s/scripts/win/build-local.ps1 -NoCache - -# Explicit cluster target if multiple exist: -./k8s/scripts/win/build-local.ps1 -ClusterName simplens -``` - ---- - -### `deploy-all.ps1` -Applies all manifests, sets up storage, reads `.env`, starts the infra pod, and brings up the chosen app environment. -```powershell -# Start with Local build active (default): -./k8s/scripts/win/deploy-all.ps1 -ActiveEnv local - -# Or start with Master (Prod GHCR) active: -./k8s/scripts/win/deploy-all.ps1 -ActiveEnv master - -# Or start with Development (GHCR) active: -./k8s/scripts/win/deploy-all.ps1 -ActiveEnv development -``` - ---- - -### `port-forward.ps1` -Tunnels ports from the Kubernetes cluster to your Windows machine (`localhost`). Automatically terminates stale `kubectl port-forward` processes and detects active NodePorts to prevent port collisions. -```powershell -# Forwards ports for ALL currently running environments + Kafka UI + Grafana: -./k8s/scripts/win/port-forward.ps1 - -# Or target only a specific environment: -./k8s/scripts/win/port-forward.ps1 -Env local -./k8s/scripts/win/port-forward.ps1 -Env master -./k8s/scripts/win/port-forward.ps1 -Env development -``` - ---- - -### `switch-env.ps1` -Switch which application environment is actively processing notifications. -*(Because all 3 environments connect to the same shared Kafka queue, running one environment at a time prevents queue collisions)*: - -```powershell -# Switch to Local build (uses existing build): -./k8s/scripts/win/switch-env.ps1 -Env local - -# Rebuild local code, load into Kind, restart pod, and wait for readiness: -./k8s/scripts/win/switch-env.ps1 -Env local -PullLatest - -# Rebuild without Docker cache: -./k8s/scripts/win/switch-env.ps1 -Env local -PullLatest -NoCache - -# Switch to Development (uses cached image): -./k8s/scripts/win/switch-env.ps1 -Env development - -# Pull latest image from GHCR on demand, then switch: -./k8s/scripts/win/switch-env.ps1 -Env development -PullLatest -./k8s/scripts/win/switch-env.ps1 -Env master -PullLatest - -# Check current pod status: -./k8s/scripts/win/switch-env.ps1 -Env status -``` - ---- - -## 4. Port Allocations Reference - -When running `./k8s/scripts/win/port-forward.ps1`, ports are mapped as follows: - -| Environment | Dashboard | API | Description | -| :--- | :--- | :--- | :--- | -| **Local** | `http://localhost:30302` *(or :3002)* | `http://localhost:30300` *(or :3000)* | Your locally built code | -| **Master** | `http://localhost:30102` | `http://localhost:30100` | Latest release from `master` branch | -| **Development** | `http://localhost:30202` | `http://localhost:30200` | Latest build from `development` branch | -| **Kafka UI** | `http://localhost:8080` | — | Inspect Kafka topics & consumers | -| **Grafana** | [http://localhost:3001](http://localhost:3001) | — | Observability & Loki logs | - ---- - -## 5. How Environment Variables Work (`.env`) - -You do not need to manually copy environment variables into Kubernetes manifests: -1. `deploy-all.ps1` reads your root `.env` file (or `.env.example` as fallback). -2. It creates a Kubernetes Secret called `app-env` inside the `simplens` namespace. -3. Every container in the App Pod automatically imports these variables via `envFrom`. -4. Cluster-internal URLs (like `mongo:27017`, `kafka:9093`, `redis:6379`) and environment-specific database names (`simplens_local`, `simplens_master`, `simplens_development`) are automatically applied as overrides so each environment stays isolated. - ---- - -## 6. Useful Debugging Commands - -```powershell -# See all pods and their status: -kubectl get pods -n simplens - -# View logs from a specific container: -kubectl logs -n simplens deploy/app-local -c api -f -kubectl logs -n simplens deploy/app-local -c worker -f -kubectl logs -n simplens deploy/app-local -c notification-processor -f -kubectl logs -n simplens deploy/simplens-infra -c mongo -f -kubectl logs -n simplens deploy/simplens-infra -c kafka -f - -# Restart a deployment: -kubectl rollout restart deployment/app-local -n simplens -kubectl rollout restart deployment/simplens-infra -n simplens -``` - ---- \ No newline at end of file diff --git a/k8s/apps/app-development.yaml b/k8s/apps/app-development.yaml deleted file mode 100644 index cdce43f..0000000 --- a/k8s/apps/app-development.yaml +++ /dev/null @@ -1,236 +0,0 @@ -# ============================================================================== -# SimpleNS Development Environment (GHCR Development Branch Images) -# Single Pod containing 6 containers: API, Worker, Unified Processor, -# Delayed Processor, Recovery Service, and Next.js Dashboard. -# ============================================================================== -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app-development - namespace: simplens - labels: - app.kubernetes.io/name: app-development - app.kubernetes.io/env: development -spec: - replicas: 0 # Start scaled down to prevent Kafka queue conflict until switched - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: app-development - template: - metadata: - labels: - app.kubernetes.io/name: app-development - app.kubernetes.io/env: development - spec: - containers: - # ============================================ - # 1. API Server - # ============================================ - - name: api - image: ghcr.io/simplenotificationsystem/simplens-core:development - imagePullPolicy: IfNotPresent - command: ["node", "dist/api/server.js"] - ports: - - containerPort: 3000 - name: http-api - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: PORT - value: "3000" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/1" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - readinessProbe: - httpGet: - path: /api/health - port: 3000 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - - # ============================================ - # 2. Outbox Worker - # ============================================ - - name: worker - image: ghcr.io/simplenotificationsystem/simplens-core:development - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/worker.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/1" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 3. Unified Notification Processor - # ============================================ - - name: notification-processor - image: ghcr.io/simplenotificationsystem/simplens-core:development - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/unified/unified.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/1" - - name: BROKERS - value: "kafka:9093" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - - name: PROCESSOR_CHANNEL - value: "all" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 4. Delayed Queue Processor - # ============================================ - - name: delayed-processor - image: ghcr.io/simplenotificationsystem/simplens-core:development - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/delayed/delayed.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/1" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 5. Recovery Service (Stuck notification recovery & cron) - # ============================================ - - name: recovery - image: ghcr.io/simplenotificationsystem/simplens-core:development - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/recovery/recovery.service.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/1" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 6. Admin Dashboard (Next.js) - # ============================================ - - name: dashboard - image: ghcr.io/simplenotificationsystem/simplens-dashboard:development - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3002 - name: http-dash - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: PORT - value: "3002" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: API_BASE_URL - value: "http://localhost:3000" - - name: WEBHOOK_HOST - value: "localhost" - - name: WEBHOOK_PORT - value: "3002" - - name: HTTPS_COOKIE - value: "false" - - volumes: - - name: simplens-config-vol - configMap: - name: simplens-config - - name: plugin-data - emptyDir: {} - - name: logs-data - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: app-development - namespace: simplens - labels: - app.kubernetes.io/name: app-development - app.kubernetes.io/env: development -spec: - type: NodePort - selector: - app.kubernetes.io/name: app-development - ports: - - name: http-api - port: 3000 - targetPort: 3000 - nodePort: 30200 - - name: http-dashboard - port: 3002 - targetPort: 3002 - nodePort: 30202 diff --git a/k8s/apps/app-local.yaml b/k8s/apps/app-local.yaml deleted file mode 100644 index 65b5ce1..0000000 --- a/k8s/apps/app-local.yaml +++ /dev/null @@ -1,236 +0,0 @@ -# ============================================================================== -# SimpleNS Local Environment (Locally Built Docker Images) -# Single Pod containing 6 containers: API, Worker, Unified Processor, -# Delayed Processor, Recovery Service, and Next.js Dashboard. -# ============================================================================== -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app-local - namespace: simplens - labels: - app.kubernetes.io/name: app-local - app.kubernetes.io/env: local -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: app-local - template: - metadata: - labels: - app.kubernetes.io/name: app-local - app.kubernetes.io/env: local - spec: - containers: - # ============================================ - # 1. API Server - # ============================================ - - name: api - image: simplens-core:local - imagePullPolicy: IfNotPresent - command: ["node", "dist/api/server.js"] - ports: - - containerPort: 3000 - name: http-api - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: PORT - value: "3000" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/2" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - readinessProbe: - httpGet: - path: /api/health - port: 3000 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - - # ============================================ - # 2. Outbox Worker - # ============================================ - - name: worker - image: simplens-core:local - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/worker.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/2" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 3. Unified Notification Processor - # ============================================ - - name: notification-processor - image: simplens-core:local - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/unified/unified.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/2" - - name: BROKERS - value: "kafka:9093" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - - name: PROCESSOR_CHANNEL - value: "all" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 4. Delayed Queue Processor - # ============================================ - - name: delayed-processor - image: simplens-core:local - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/delayed/delayed.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/2" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 5. Recovery Service (Stuck notification recovery & cron) - # ============================================ - - name: recovery - image: simplens-core:local - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/recovery/recovery.service.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/2" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 6. Admin Dashboard (Next.js) - # ============================================ - - name: dashboard - image: simplens-dashboard:local - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3002 - name: http-dash - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: PORT - value: "3002" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: API_BASE_URL - value: "http://localhost:3000" - - name: WEBHOOK_HOST - value: "localhost" - - name: WEBHOOK_PORT - value: "3002" - - name: HTTPS_COOKIE - value: "false" - - volumes: - - name: simplens-config-vol - configMap: - name: simplens-config - - name: plugin-data - emptyDir: {} - - name: logs-data - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: app-local - namespace: simplens - labels: - app.kubernetes.io/name: app-local - app.kubernetes.io/env: local -spec: - type: NodePort - selector: - app.kubernetes.io/name: app-local - ports: - - name: http-api - port: 3000 - targetPort: 3000 - nodePort: 30300 - - name: http-dashboard - port: 3002 - targetPort: 3002 - nodePort: 30302 diff --git a/k8s/apps/app-master.yaml b/k8s/apps/app-master.yaml deleted file mode 100644 index 3ab2d0b..0000000 --- a/k8s/apps/app-master.yaml +++ /dev/null @@ -1,236 +0,0 @@ -# ============================================================================== -# SimpleNS Master Environment (Production - GHCR Master Images) -# Single Pod containing 6 containers: API, Worker, Unified Processor, -# Delayed Processor, Recovery Service, and Next.js Dashboard. -# ============================================================================== -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app-master - namespace: simplens - labels: - app.kubernetes.io/name: app-master - app.kubernetes.io/env: master -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: app-master - template: - metadata: - labels: - app.kubernetes.io/name: app-master - app.kubernetes.io/env: master - spec: - containers: - # ============================================ - # 1. API Server - # ============================================ - - name: api - image: ghcr.io/simplenotificationsystem/simplens-core:latest - imagePullPolicy: IfNotPresent - command: ["node", "dist/api/server.js"] - ports: - - containerPort: 3000 - name: http-api - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: PORT - value: "3000" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/0" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - readinessProbe: - httpGet: - path: /api/health - port: 3000 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - - # ============================================ - # 2. Outbox Worker - # ============================================ - - name: worker - image: ghcr.io/simplenotificationsystem/simplens-core:latest - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/worker.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/0" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 3. Unified Notification Processor - # ============================================ - - name: notification-processor - image: ghcr.io/simplenotificationsystem/simplens-core:latest - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/unified/unified.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/0" - - name: BROKERS - value: "kafka:9093" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - - name: PROCESSOR_CHANNEL - value: "all" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 4. Delayed Queue Processor - # ============================================ - - name: delayed-processor - image: ghcr.io/simplenotificationsystem/simplens-core:latest - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/delayed/delayed.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/0" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 5. Recovery Service (Stuck notification recovery & cron) - # ============================================ - - name: recovery - image: ghcr.io/simplenotificationsystem/simplens-core:latest - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/recovery/recovery.service.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/0" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 6. Admin Dashboard (Next.js) - # ============================================ - - name: dashboard - image: ghcr.io/simplenotificationsystem/simplens-dashboard:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3002 - name: http-dash - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: PORT - value: "3002" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: API_BASE_URL - value: "http://localhost:3000" - - name: WEBHOOK_HOST - value: "localhost" - - name: WEBHOOK_PORT - value: "3002" - - name: HTTPS_COOKIE - value: "false" - - volumes: - - name: simplens-config-vol - configMap: - name: simplens-config - - name: plugin-data - emptyDir: {} - - name: logs-data - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: app-master - namespace: simplens - labels: - app.kubernetes.io/name: app-master - app.kubernetes.io/env: master -spec: - type: NodePort - selector: - app.kubernetes.io/name: app-master - ports: - - name: http-api - port: 3000 - targetPort: 3000 - nodePort: 30100 - - name: http-dashboard - port: 3002 - targetPort: 3002 - nodePort: 30102 diff --git a/k8s/base/configmap-simplens.yaml b/k8s/base/configmap-simplens.yaml deleted file mode 100644 index 3e0c2d4..0000000 --- a/k8s/base/configmap-simplens.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: simplens-config - namespace: simplens -data: - simplens.config.yaml: | - # ============================================ - # SimpleNS Configuration - # ============================================ - providers: - # mock - @simplens/mock - - package: "@simplens/mock" - id: mock - credentials: - {} # No credentials required - options: - priority: 1 # Higher = preferred - rateLimit: - maxTokens: 100 # Token bucket capacity - refillRate: 10 # Tokens per interval - refillInterval: second # Options: second, minute, hour, day - - # mock-fb - @simplens/mock - - package: "@simplens/mock" - id: mock-fb - credentials: - {} # No credentials required - options: - priority: 1 # Higher = preferred - rateLimit: - maxTokens: 100 # Token bucket capacity - refillRate: 10 # Tokens per interval - refillInterval: second # Options: second, minute, hour, day - - # nodemailer-gmail - @simplens/nodemailer-gmail - - package: "@simplens/nodemailer-gmail" - id: nodemailer-gmail - credentials: - EMAIL_USER: ${EMAIL_USER} - EMAIL_PASS: ${EMAIL_PASS} - optionalConfig: - EMAIL_HOST: ${EMAIL_HOST} - EMAIL_PORT: ${EMAIL_PORT} - EMAIL_FROM: ${EMAIL_FROM} - options: - priority: 1 - rateLimit: - maxTokens: 100 - refillRate: 10 - refillInterval: second - - channels: - mock: - default: mock - fallback: mock-fb - email: - default: nodemailer-gmail diff --git a/k8s/base/namespace.yaml b/k8s/base/namespace.yaml deleted file mode 100644 index 1df6ef8..0000000 --- a/k8s/base/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: simplens - labels: - app.kubernetes.io/name: simplens diff --git a/k8s/infra/pod-infra.yaml b/k8s/infra/pod-infra.yaml deleted file mode 100644 index 6aae517..0000000 --- a/k8s/infra/pod-infra.yaml +++ /dev/null @@ -1,201 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplens-infra - namespace: simplens - labels: - app.kubernetes.io/name: simplens-infra - app.kubernetes.io/component: infrastructure -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: simplens-infra - template: - metadata: - labels: - app.kubernetes.io/name: simplens-infra - app.kubernetes.io/component: infrastructure - spec: - hostAliases: - - ip: "127.0.0.1" - hostnames: - - "mongo" - - "kafka" - - "redis" - - "loki" - containers: - # ============================================ - # 1. MongoDB (7.0 with replica set rs0) - # ============================================ - - name: mongo - image: mongo:7.0 - command: - - mongod - - --replSet - - rs0 - - --bind_ip_all - - --port - - "27017" - ports: - - containerPort: 27017 - name: mongo - volumeMounts: - - name: mongo-data - mountPath: /data/db - - name: mongo-config - mountPath: /data/configdb - readinessProbe: - exec: - command: - - mongosh - - --port - - "27017" - - --quiet - - --eval - - "try { if (rs.status().ok === 1) quit(0); } catch(e) {} try { rs.initiate({_id:'rs0',members:[{_id:0,host:'mongo:27017'}]}); quit(0); } catch(err) { quit(1); }" - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 10 - failureThreshold: 10 - - # ============================================ - # 2. Apache Kafka (native KRaft mode) - # ============================================ - - name: kafka - image: apache/kafka-native - ports: - - containerPort: 9091 - name: controller - - containerPort: 9092 - name: host - - containerPort: 9093 - name: internal - env: - - name: KAFKA_LISTENERS - value: "CONTROLLER://localhost:9091,INTERNAL://0.0.0.0:9093,HOST://0.0.0.0:9092" - - name: KAFKA_ADVERTISED_LISTENERS - value: "INTERNAL://kafka:9093,HOST://localhost:9092" - - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP - value: "CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,HOST:PLAINTEXT" - - name: KAFKA_NODE_ID - value: "1" - - name: KAFKA_PROCESS_ROLES - value: "broker,controller" - - name: KAFKA_CONTROLLER_LISTENER_NAMES - value: "CONTROLLER" - - name: KAFKA_CONTROLLER_QUORUM_VOTERS - value: "1@localhost:9091" - - name: KAFKA_INTER_BROKER_LISTENER_NAME - value: "INTERNAL" - - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR - value: "1" - - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE - value: "false" - volumeMounts: - - name: kafka-data - mountPath: /var/lib/kafka/data - - # ============================================ - # 3. Kafka UI (Management dashboard) - # ============================================ - - name: kafka-ui - image: kafbat/kafka-ui:main - ports: - - containerPort: 8080 - name: http-kafkaui - env: - - name: DYNAMIC_CONFIG_ENABLED - value: "true" - - name: KAFKA_CLUSTERS_0_NAME - value: "local" - - name: KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS - value: "localhost:9093" - - # ============================================ - # 4. Redis (7-alpine with appendonly) - # ============================================ - - name: redis - image: redis:7-alpine - command: - - redis-server - - --appendonly - - "yes" - ports: - - containerPort: 6379 - name: redis - volumeMounts: - - name: redis-data - mountPath: /data - readinessProbe: - exec: - command: - - redis-cli - - ping - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 3 - - # ============================================ - # 5. Loki (Log aggregation) - # ============================================ - - name: loki - image: grafana/loki:2.9.0 - args: - - -config.file=/etc/loki/local-config.yaml - ports: - - containerPort: 3100 - name: http-loki - volumeMounts: - - name: loki-data - mountPath: /loki - readinessProbe: - httpGet: - path: /ready - port: 3100 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - - # ============================================ - # 6. Grafana (Dashboard & observability) - # ============================================ - - name: grafana - image: grafana/grafana:10.2.0 - ports: - - containerPort: 3000 - name: http-grafana - env: - - name: GF_PATHS_PROVISIONING - value: "/etc/grafana/provisioning" - - name: GF_AUTH_ANONYMOUS_ENABLED - value: "true" - - name: GF_AUTH_ANONYMOUS_ORG_ROLE - value: "Admin" - - name: GF_SECURITY_ADMIN_PASSWORD - value: "admin" - volumeMounts: - - name: grafana-data - mountPath: /var/lib/grafana - - volumes: - - name: mongo-data - persistentVolumeClaim: - claimName: mongo-data - - name: mongo-config - persistentVolumeClaim: - claimName: mongo-config - - name: kafka-data - persistentVolumeClaim: - claimName: kafka-data - - name: redis-data - persistentVolumeClaim: - claimName: redis-data - - name: loki-data - persistentVolumeClaim: - claimName: loki-data - - name: grafana-data - persistentVolumeClaim: - claimName: grafana-data diff --git a/k8s/infra/pvc-infra.yaml b/k8s/infra/pvc-infra.yaml deleted file mode 100644 index f41062a..0000000 --- a/k8s/infra/pvc-infra.yaml +++ /dev/null @@ -1,71 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: mongo-data - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: mongo-config - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 500Mi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: kafka-data - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: redis-data - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: loki-data - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: grafana-data - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/k8s/infra/services-infra.yaml b/k8s/infra/services-infra.yaml deleted file mode 100644 index 3854a73..0000000 --- a/k8s/infra/services-infra.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# ============================================================================== -# Internal ClusterIP Services for Intra-Cluster Communication -# Allows all App Pods to reach MongoDB, Kafka, Redis, and Loki by name -# ============================================================================== -apiVersion: v1 -kind: Service -metadata: - name: mongo - namespace: simplens - labels: - app.kubernetes.io/name: mongo -spec: - type: NodePort - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: mongo - port: 27017 - targetPort: 27017 - nodePort: 30717 ---- -apiVersion: v1 -kind: Service -metadata: - name: kafka - namespace: simplens - labels: - app.kubernetes.io/name: kafka -spec: - type: NodePort - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: internal - port: 9093 - targetPort: 9093 - - name: host - port: 9092 - targetPort: 9092 - nodePort: 30092 ---- -apiVersion: v1 -kind: Service -metadata: - name: redis - namespace: simplens - labels: - app.kubernetes.io/name: redis -spec: - type: NodePort - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: redis - port: 6379 - targetPort: 6379 - nodePort: 30379 ---- -apiVersion: v1 -kind: Service -metadata: - name: loki - namespace: simplens - labels: - app.kubernetes.io/name: loki -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: http-loki - port: 3100 - targetPort: 3100 ---- -# ============================================================================== -# Host-Facing NodePort Services for Web Dashboards -# ============================================================================== -apiVersion: v1 -kind: Service -metadata: - name: kafka-ui - namespace: simplens - labels: - app.kubernetes.io/name: kafka-ui -spec: - type: NodePort - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: http-kafkaui - port: 8080 - targetPort: 8080 - nodePort: 30080 ---- -apiVersion: v1 -kind: Service -metadata: - name: grafana - namespace: simplens - labels: - app.kubernetes.io/name: grafana -spec: - type: NodePort - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: http-grafana - port: 3000 - targetPort: 3000 - nodePort: 30001 diff --git a/k8s/kind-config.yaml b/k8s/kind-config.yaml deleted file mode 100644 index eedba61..0000000 --- a/k8s/kind-config.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# ============================================================================== -# Kind Cluster Configuration with Port Mappings for NodePorts -# Use this configuration when creating a cluster with: -# kind create cluster --name simplens --config k8s/kind-config.yaml -# ============================================================================== -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: - - role: control-plane - extraPortMappings: - # ============================================ - # Infrastructure Services - # ============================================ - - containerPort: 30001 - hostPort: 30001 - listenAddress: "127.0.0.1" # Grafana (Host: http://localhost:30001) - - containerPort: 30080 - hostPort: 30080 - listenAddress: "127.0.0.1" # Kafka-UI (Host: http://localhost:30080) - - containerPort: 30092 - hostPort: 30092 - listenAddress: "127.0.0.1" # Kafka external host listener - - containerPort: 30379 - hostPort: 30379 - listenAddress: "127.0.0.1" # Redis - - containerPort: 30717 - hostPort: 30717 - listenAddress: "127.0.0.1" # MongoDB - - # ============================================ - # App Master (Prod - GHCR Master) - # ============================================ - - containerPort: 30100 - hostPort: 30100 - listenAddress: "127.0.0.1" # Master API (Host: http://localhost:30100) - - containerPort: 30102 - hostPort: 30102 - listenAddress: "127.0.0.1" # Master Dashboard (Host: http://localhost:30102) - - # ============================================ - # App Development (GHCR Dev Branch) - # ============================================ - - containerPort: 30200 - hostPort: 30200 - listenAddress: "127.0.0.1" # Development API (Host: http://localhost:30200) - - containerPort: 30202 - hostPort: 30202 - listenAddress: "127.0.0.1" # Development Dashboard (Host: http://localhost:30202) - - # ============================================ - # App Local (Locally Built Image) - # ============================================ - - containerPort: 30300 - hostPort: 30300 - listenAddress: "127.0.0.1" # Local API (Host: http://localhost:30300) - - containerPort: 30302 - hostPort: 30302 - listenAddress: "127.0.0.1" # Local Dashboard (Host: http://localhost:30302) diff --git a/k8s/scripts/win/build-local.ps1 b/k8s/scripts/win/build-local.ps1 deleted file mode 100644 index 8fd823e..0000000 --- a/k8s/scripts/win/build-local.ps1 +++ /dev/null @@ -1,92 +0,0 @@ -<# -.SYNOPSIS - Builds the local SimpleNS Core and Dashboard Docker images and loads them into Kind (if using Kind). - -.PARAMETER ClusterName - The name of the Kind cluster (default: "simplens"). - -.PARAMETER LoadKind - Switch to load the built images into Kind using 'kind load docker-image'. -#> -param( - [string]$ClusterName = "simplens", - [switch]$LoadKind, - [switch]$NoCache -) - -$ErrorActionPreference = "Stop" -$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path -$K8sDir = Split-Path -Parent (Split-Path -Parent $ScriptDir) -$RootDir = Split-Path -Parent $K8sDir - -$BuildFlags = if ($NoCache) { "--no-cache" } else { "" } - -Write-Host "==========================================" -ForegroundColor Cyan -Write-Host " Building Local SimpleNS Docker Images " -ForegroundColor Cyan -Write-Host "==========================================" -ForegroundColor Cyan - -# 1. Build simplens-core:local -Write-Host "`n[1/2] Building simplens-core:local from root Dockerfile..." -ForegroundColor Yellow -docker build -t simplens-core:local -f "$RootDir/Dockerfile" "$RootDir" $BuildFlags -if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to build simplens-core:local" -} -Write-Host "[OK] simplens-core:local built successfully!" -ForegroundColor Green - -# 2. Build simplens-dashboard:local -Write-Host "`n[2/2] Building simplens-dashboard:local from dashboard/Dockerfile..." -ForegroundColor Yellow -docker build -t simplens-dashboard:local -f "$RootDir/dashboard/Dockerfile" "$RootDir/dashboard" $BuildFlags -if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to build simplens-dashboard:local" -} -Write-Host "[OK] simplens-dashboard:local built successfully!" -ForegroundColor Green - -# 3. Load into Kind if requested or if Kind cluster is detected -$TargetKindCluster = $ClusterName -$ShouldLoadKind = $LoadKind.IsPresent - -if (-not $ShouldLoadKind) { - if (Get-Command kind -ErrorAction SilentlyContinue) { - $rawClusters = (kind get clusters 2>$null) - if ($rawClusters) { - $clusters = ($rawClusters -split "`r?`n") | Where-Object { $_.Trim() -ne "" } | ForEach-Object { $_.Trim() } - - # Check if current kubectl context indicates a kind cluster - $currentContext = (kubectl config current-context 2>$null) - if ($currentContext -and $currentContext -like "kind-*") { - $contextCluster = $currentContext.Substring(5) - if ($clusters -contains $contextCluster) { - $TargetKindCluster = $contextCluster - $ShouldLoadKind = $true - } - } - - if (-not $ShouldLoadKind) { - if ($clusters -contains $ClusterName) { - $TargetKindCluster = $ClusterName - $ShouldLoadKind = $true - } elseif ($clusters.Count -eq 1) { - $TargetKindCluster = $clusters[0] - $ShouldLoadKind = $true - } - } - } - } -} - -if ($ShouldLoadKind) { - Write-Host "`n[Kind] Loading images into Kind cluster '$TargetKindCluster'..." -ForegroundColor Cyan - kind load docker-image simplens-core:local --name $TargetKindCluster - kind load docker-image simplens-dashboard:local --name $TargetKindCluster - Write-Host "[OK] Images loaded into Kind cluster '$TargetKindCluster'!" -ForegroundColor Green -} else { - Write-Host "`nNote: If using Docker Desktop Kubernetes, local Docker images are shared automatically." -ForegroundColor DarkGray - Write-Host "If using Kind, pass -LoadKind or run: kind load docker-image simplens-core:local --name $ClusterName" -ForegroundColor DarkGray -} - -# 4. Clean up dangling images to avoid duplicate : images in Docker -Write-Host "`nCleaning up dangling Docker images..." -ForegroundColor Cyan -docker image prune -f --filter "dangling=true" 2>$null | Out-Null -Write-Host "[OK] Dangling images cleaned up." -ForegroundColor Green - -Write-Host "`nAll images are ready for Kubernetes!" -ForegroundColor Green diff --git a/k8s/scripts/win/deploy-all.ps1 b/k8s/scripts/win/deploy-all.ps1 deleted file mode 100644 index eb5160b..0000000 --- a/k8s/scripts/win/deploy-all.ps1 +++ /dev/null @@ -1,77 +0,0 @@ -<# -.SYNOPSIS - Deploys the complete SimpleNS Kubernetes stack (Infra Pod + App Pods) to the local cluster. - -.PARAMETER ActiveEnv - The initial active app environment to run: 'master', 'development', or 'local' (default: 'local'). -#> -param( - [ValidateSet("master", "development", "local", "all")] - [string]$ActiveEnv = "local" -) - -$ErrorActionPreference = "Stop" -$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path -$K8sDir = Split-Path -Parent (Split-Path -Parent $ScriptDir) -$RootDir = Split-Path -Parent $K8sDir - -Write-Host "==========================================" -ForegroundColor Cyan -Write-Host " Deploying SimpleNS Multi-Env Kubernetes " -ForegroundColor Cyan -Write-Host "==========================================" -ForegroundColor Cyan - -# 1. Namespace -Write-Host "`n[1/7] Applying namespace..." -ForegroundColor Yellow -kubectl apply -f "$K8sDir/base/namespace.yaml" - -# 2. ConfigMap -Write-Host "`n[2/7] Applying simplens configuration ConfigMap..." -ForegroundColor Yellow -kubectl apply -f "$K8sDir/base/configmap-simplens.yaml" - -# 3. Environment Variables Secret (.env injection) -Write-Host "`n[3/7] Injecting .env file into Kubernetes Secret 'app-env'..." -ForegroundColor Yellow -$EnvPath = "$RootDir/.env" -if (-not (Test-Path $EnvPath)) { - Write-Warning ".env file not found at '$EnvPath'. Falling back to '.env.example'." - $EnvPath = "$RootDir/.env.example" -} -# Create or update Secret idempotently using --dry-run -kubectl create secret generic app-env --from-env-file="$EnvPath" --namespace simplens --dry-run=client -o yaml | kubectl apply -f - -Write-Host "[OK] Secret 'app-env' created/updated from $EnvPath" -ForegroundColor Green - -# 4. Persistent Volumes -Write-Host "`n[4/7] Applying persistent volume claims..." -ForegroundColor Yellow -kubectl apply -f "$K8sDir/infra/pvc-infra.yaml" - -# 5. Infrastructure Pod & Services -Write-Host "`n[5/7] Deploying infrastructure pod (Mongo, Kafka, Redis, Loki, Grafana, Kafka-UI)..." -ForegroundColor Yellow -kubectl apply -f "$K8sDir/infra/pod-infra.yaml" -kubectl apply -f "$K8sDir/infra/services-infra.yaml" - -Write-Host "`nWaiting for infrastructure pod to be ready..." -ForegroundColor Cyan -kubectl wait --namespace simplens --for=condition=ready pod -l app.kubernetes.io/name=simplens-infra --timeout=120s - -# 6. App Deployments -Write-Host "`n[6/7] Deploying application environments (Master, Development, Local)..." -ForegroundColor Yellow -kubectl apply -f "$K8sDir/apps/app-master.yaml" -kubectl apply -f "$K8sDir/apps/app-development.yaml" -kubectl apply -f "$K8sDir/apps/app-local.yaml" - -# 7. Set Active Environment -Write-Host "`n[7/7] Activating '$ActiveEnv' environment..." -ForegroundColor Yellow -& "$ScriptDir/switch-env.ps1" -Env $ActiveEnv - -Write-Host "`n==========================================" -ForegroundColor Green -Write-Host " Deployment Complete! " -ForegroundColor Green -Write-Host "==========================================" -ForegroundColor Green -Write-Host "Active App Environment: $ActiveEnv" -ForegroundColor Cyan -Write-Host "`nAccess URLs (on localhost via port-forward):" -ForegroundColor White -Write-Host " - Active Dashboard: http://localhost:3002 (or 30302 for Local)" -ForegroundColor White -Write-Host " - Active API: http://localhost:3000 (or 30300 for Local)" -ForegroundColor White -Write-Host " - Kafka UI: http://localhost:8080" -ForegroundColor White -Write-Host " - Grafana: http://localhost:3001 (admin / admin)" -ForegroundColor White -Write-Host "`nStart port-forwarding by running:" -ForegroundColor DarkGray -Write-Host " ./k8s/scripts/win/port-forward.ps1" -ForegroundColor DarkGray -Write-Host "`nTo switch testing environments, run:" -ForegroundColor DarkGray -Write-Host " ./k8s/scripts/win/switch-env.ps1 -Env master" -ForegroundColor DarkGray -Write-Host " ./k8s/scripts/win/switch-env.ps1 -Env development" -ForegroundColor DarkGray -Write-Host " ./k8s/scripts/win/switch-env.ps1 -Env local" -ForegroundColor DarkGray diff --git a/k8s/scripts/win/port-forward.ps1 b/k8s/scripts/win/port-forward.ps1 deleted file mode 100644 index ad43ce2..0000000 --- a/k8s/scripts/win/port-forward.ps1 +++ /dev/null @@ -1,154 +0,0 @@ -<# -.SYNOPSIS - Tunnels ports from the Kubernetes cluster to your Windows host machine (localhost). - Supports forwarding for a specific environment OR all active environments simultaneously. - -.PARAMETER Env - The app environment to forward: 'all', 'local', 'master', or 'development' (default: 'all'). -#> -param( - [ValidateSet("all", "local", "master", "development")] - [string]$Env = "all" -) - -$Namespace = "simplens" - -Write-Host "==========================================" -ForegroundColor Cyan -Write-Host " SimpleNS Multi-Env Port Forwarding " -ForegroundColor Cyan -Write-Host "==========================================" -ForegroundColor Cyan - -# Stop any existing kubectl port-forward processes reliably across PowerShell versions -Get-CimInstance Win32_Process -Filter "Name = 'kubectl.exe'" -ErrorAction SilentlyContinue | - Where-Object { $_.CommandLine -like "*port-forward*" } | - ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } - -function Test-PortInUse([int]$port) { - try { - $tcp = New-Object System.Net.Sockets.TcpClient - $iar = $tcp.BeginConnect("127.0.0.1", $port, $null, $null) - $success = $iar.AsyncWaitHandle.WaitOne(200, $false) - if ($success) { - $tcp.EndConnect($iar) - $tcp.Close() - return $true - } - $tcp.Close() - return $false - } catch { - return $false - } -} - -$jobs = @() - -# 1. Forward Kafka-UI (8080) -if (-not (Test-PortInUse 8080)) { - $jobs += Start-Job -ScriptBlock { - param($ns) - while ($true) { - kubectl port-forward -n $ns svc/kafka-ui 8080:8080 2>$null - Start-Sleep -Milliseconds 800 - } - } -ArgumentList $Namespace -} - -# 2. Forward Grafana (3001 -> 3000) -if (-not (Test-PortInUse 3001)) { - $jobs += Start-Job -ScriptBlock { - param($ns) - while ($true) { - kubectl port-forward -n $ns svc/grafana 3001:3000 2>$null - Start-Sleep -Milliseconds 800 - } - } -ArgumentList $Namespace -} - -# Helper function to check if a deployment is running -function Is-Running($deploymentName) { - $replicas = (kubectl get deployment $deploymentName -n $Namespace -o jsonpath='{.spec.replicas}' 2>$null) - return ($replicas -and [int]$replicas -gt 0) -} - -# 3. Forward Local Environment (Ports 3000/3002 and NodePorts 30300/30302) -if (($Env -eq "all" -or $Env -eq "local") -and (Is-Running "app-local")) { - $localPorts = @() - if (-not (Test-PortInUse 3000)) { $localPorts += "3000:3000" } - if (-not (Test-PortInUse 3002)) { $localPorts += "3002:3002" } - if (-not (Test-PortInUse 30300)) { $localPorts += "30300:3000" } - if (-not (Test-PortInUse 30302)) { $localPorts += "30302:3002" } - - if ($localPorts.Count -gt 0) { - $pStr = $localPorts -join " " - $jobs += Start-Job -ScriptBlock { - param($ns, $p) - while ($true) { - Invoke-Expression "kubectl port-forward -n $ns svc/app-local $p 2>`$null" - Start-Sleep -Milliseconds 800 - } - } -ArgumentList $Namespace, $pStr - } - Write-Host "[+] Local App Access (Auto-reconnecting):" -ForegroundColor Green - Write-Host " - Dashboard: http://localhost:3002 (NodePort: http://localhost:30302)" -ForegroundColor White - Write-Host " - API: http://localhost:3000 (NodePort: http://localhost:30300)" -ForegroundColor White -} - -# 4. Forward Master Environment (Ports 30100/30102) -if (($Env -eq "all" -or $Env -eq "master") -and (Is-Running "app-master")) { - $masterPorts = @() - if (-not (Test-PortInUse 30100)) { $masterPorts += "30100:3000" } - if (-not (Test-PortInUse 30102)) { $masterPorts += "30102:3002" } - - if ($masterPorts.Count -gt 0) { - $pStr = $masterPorts -join " " - $jobs += Start-Job -ScriptBlock { - param($ns, $p) - while ($true) { - Invoke-Expression "kubectl port-forward -n $ns svc/app-master $p 2>`$null" - Start-Sleep -Milliseconds 800 - } - } -ArgumentList $Namespace, $pStr - } - Write-Host "[+] Master (Prod) App Access (Auto-reconnecting):" -ForegroundColor Green - Write-Host " - Dashboard: http://localhost:30102" -ForegroundColor White - Write-Host " - API: http://localhost:30100" -ForegroundColor White -} - -# 5. Forward Development Branch Environment (Ports 30200/30202) -if (($Env -eq "all" -or $Env -eq "development") -and (Is-Running "app-development")) { - $devPorts = @() - if (-not (Test-PortInUse 30200)) { $devPorts += "30200:3000" } - if (-not (Test-PortInUse 30202)) { $devPorts += "30202:3002" } - - if ($devPorts.Count -gt 0) { - $pStr = $devPorts -join " " - $jobs += Start-Job -ScriptBlock { - param($ns, $p) - while ($true) { - Invoke-Expression "kubectl port-forward -n $ns svc/app-development $p 2>`$null" - Start-Sleep -Milliseconds 800 - } - } -ArgumentList $Namespace, $pStr - } - Write-Host "[+] Development Branch App Access (Auto-reconnecting):" -ForegroundColor Green - Write-Host " - Dashboard: http://localhost:30202" -ForegroundColor White - Write-Host " - API: http://localhost:30200" -ForegroundColor White -} - -Start-Sleep -Seconds 2 - -Write-Host "`n[+] Infrastructure Services:" -ForegroundColor Green -Write-Host " - Kafka UI: http://localhost:8080" -ForegroundColor White -Write-Host " - Grafana: http://localhost:3001 (admin / admin)" -ForegroundColor White - -Write-Host "`n[OK] Port forwarding is active! Press Ctrl+C to close all tunnels.`n" -ForegroundColor Cyan - -try { - while ($true) { - Start-Sleep -Seconds 1 - } -} finally { - Write-Host "`nClosing port tunnels..." -ForegroundColor Yellow - $jobs | Stop-Job -ErrorAction SilentlyContinue - $jobs | Remove-Job -ErrorAction SilentlyContinue - Write-Host "[OK] Tunnels closed." -ForegroundColor Green -} diff --git a/k8s/scripts/win/switch-env.ps1 b/k8s/scripts/win/switch-env.ps1 deleted file mode 100644 index 1649c1b..0000000 --- a/k8s/scripts/win/switch-env.ps1 +++ /dev/null @@ -1,155 +0,0 @@ -<# -.SYNOPSIS - Switches the active SimpleNS application environment (Master, Development, Local, or All). - Ensures that multiple environments do not contend on shared Kafka consumer groups. - -.PARAMETER Env - The environment to activate: 'master', 'development', 'local', 'all', or 'status'. - -.PARAMETER PullLatest - If specified, pulls the latest images from GHCR (for master/development) or rebuilds local images (for local) before running. -#> -param( - [ValidateSet("master", "development", "local", "all", "status")] - [string]$Env = "status", - - [switch]$PullLatest, - [switch]$NoCache -) - -$Namespace = "simplens" -$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path - -function Show-Status { - Write-Host "`n--- Current SimpleNS Pod Status ---" -ForegroundColor Cyan - kubectl get pods -n $Namespace -o wide - Write-Host "`n--- App Deployments Replica Counts ---" -ForegroundColor Cyan - kubectl get deployments -n $Namespace -l "app.kubernetes.io/name in (app-master, app-development, app-local)" -} - -function Sync-ToKind { - param([string[]]$Images) - if (Get-Command kind -ErrorAction SilentlyContinue) { - $rawClusters = (kind get clusters 2>$null) - if ($rawClusters) { - $clusters = ($rawClusters -split "`r?`n") | Where-Object { $_.Trim() -ne "" } | ForEach-Object { $_.Trim() } - $targetCluster = $null - $currentContext = (kubectl config current-context 2>$null) - if ($currentContext -and $currentContext -like "kind-*") { - $contextCluster = $currentContext.Substring(5) - if ($clusters -contains $contextCluster) { - $targetCluster = $contextCluster - } - } - if (-not $targetCluster) { - if ($clusters -contains "simplens") { - $targetCluster = "simplens" - } elseif ($clusters.Count -eq 1) { - $targetCluster = $clusters[0] - } - } - if ($targetCluster) { - foreach ($img in $Images) { - Write-Host "[Kind] Loading image '$img' into cluster '$targetCluster'..." -ForegroundColor Cyan - kind load docker-image $img --name $targetCluster - } - } - } - } -} - -function Pull-DevelopmentImages { - Write-Host "`n[Pull] Pulling latest development images from GHCR..." -ForegroundColor Cyan - docker pull ghcr.io/simplenotificationsystem/simplens-core:development - docker pull ghcr.io/simplenotificationsystem/simplens-dashboard:development - Write-Host "[OK] Latest development images pulled." -ForegroundColor Green - Sync-ToKind @("ghcr.io/simplenotificationsystem/simplens-core:development", "ghcr.io/simplenotificationsystem/simplens-dashboard:development") -} - -function Pull-MasterImages { - Write-Host "`n[Pull] Pulling latest master images from GHCR..." -ForegroundColor Cyan - docker pull ghcr.io/simplenotificationsystem/simplens-core:latest - docker pull ghcr.io/simplenotificationsystem/simplens-dashboard:latest - Write-Host "[OK] Latest master images pulled." -ForegroundColor Green - Sync-ToKind @("ghcr.io/simplenotificationsystem/simplens-core:latest", "ghcr.io/simplenotificationsystem/simplens-dashboard:latest") -} - -switch ($Env) { - "master" { - if ($PullLatest) { - Pull-MasterImages - } - Write-Host "Activating 'master' environment (GHCR master branch)..." -ForegroundColor Yellow - kubectl scale deployment app-master -n $Namespace --replicas=1 - kubectl scale deployment app-development -n $Namespace --replicas=0 - kubectl scale deployment app-local -n $Namespace --replicas=0 - if ($PullLatest) { - kubectl rollout restart deployment app-master -n $Namespace - Write-Host "Waiting for app-master rollout to complete..." -ForegroundColor Cyan - kubectl rollout status deployment app-master -n $Namespace --timeout=120s - } - Write-Host "[OK] Switched to 'master' environment!" -ForegroundColor Green - Write-Host " API: http://localhost:30100" -ForegroundColor White - Write-Host " Dashboard: http://localhost:30102" -ForegroundColor White - } - "development" { - if ($PullLatest) { - Pull-DevelopmentImages - } - Write-Host "Activating 'development' environment (GHCR dev branch)..." -ForegroundColor Yellow - kubectl scale deployment app-master -n $Namespace --replicas=0 - kubectl scale deployment app-development -n $Namespace --replicas=1 - kubectl scale deployment app-local -n $Namespace --replicas=0 - if ($PullLatest) { - kubectl rollout restart deployment app-development -n $Namespace - Write-Host "Waiting for app-development rollout to complete..." -ForegroundColor Cyan - kubectl rollout status deployment app-development -n $Namespace --timeout=120s - } - Write-Host "[OK] Switched to 'development' environment!" -ForegroundColor Green - Write-Host " API: http://localhost:30200" -ForegroundColor White - Write-Host " Dashboard: http://localhost:30202" -ForegroundColor White - } - "local" { - if ($PullLatest) { - Write-Host "`n[Build] Rebuilding local images..." -ForegroundColor Cyan - $buildArgs = @() - if ($NoCache) { $buildArgs += "-NoCache" } - & "$ScriptDir/build-local.ps1" @buildArgs - } - Write-Host "Activating 'local' environment (locally built image)..." -ForegroundColor Yellow - kubectl scale deployment app-master -n $Namespace --replicas=0 - kubectl scale deployment app-development -n $Namespace --replicas=0 - kubectl scale deployment app-local -n $Namespace --replicas=1 - if ($PullLatest) { - Write-Host "Restarting deployment app-local..." -ForegroundColor Cyan - kubectl rollout restart deployment app-local -n $Namespace - Write-Host "Waiting for app-local rollout to complete..." -ForegroundColor Cyan - kubectl rollout status deployment app-local -n $Namespace --timeout=120s - } - Write-Host "[OK] Switched to 'local' environment!" -ForegroundColor Green - Write-Host " API: http://localhost:30300" -ForegroundColor White - Write-Host " Dashboard: http://localhost:30302" -ForegroundColor White - } - "all" { - if ($PullLatest) { - Pull-MasterImages - Pull-DevelopmentImages - } - Write-Host "Scaling all environments to 1 replica..." -ForegroundColor Yellow - Write-Warning "Notice: All 3 app environments share the same Kafka broker. Because consumer group IDs are shared, consumers will balance partitions across environments." - kubectl scale deployment app-master -n $Namespace --replicas=1 - kubectl scale deployment app-development -n $Namespace --replicas=1 - kubectl scale deployment app-local -n $Namespace --replicas=1 - if ($PullLatest) { - kubectl rollout restart deployment app-master -n $Namespace - kubectl rollout restart deployment app-development -n $Namespace - } - Write-Host "[OK] All environments scaled to 1!" -ForegroundColor Green - } - "status" { - Show-Status - return - } -} - -Show-Status diff --git a/scripts/README.md b/scripts/README.md index f22ccca..65ab842 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,14 +1,11 @@ # SimpleNS Testing & Utility Scripts -This directory contains developer utilities for testing, load generation, webhook verification, and resilience/chaos testing for SimpleNS. +This directory contains developer utilities for testing, load generation, webhook verification for SimpleNS. ``` scripts/ ├── client.js # Lightweight Express webhook receiver for callback inspection ├── load-test.js # Universal notification sender & high-concurrency load tester -├── crash-tests/ -│ ├── k8s-chaos.ps1 # Kubernetes-native container & pod chaos testing -│ └── crash-multiwave-all.sh # Docker Compose infrastructure chaos testing (8 waves) └── README.md # This documentation ``` @@ -116,95 +113,4 @@ node scripts/load-test.js -n 1000 -c 50 -ch mock -p mock -h docker ``` *Reports throughput (req/sec), min/avg/max latency, `p50`, `p95`, `p99` percentiles, and HTTP status code distributions.* ---- - -## 3. Chaos & Resilience Testing (`crash-tests/`) - -Simulate service crashes, database outages, connection drops, and container evictions under active load to verify outbox recovery and idempotency. - -### A. Kubernetes Native Chaos (`k8s-chaos.ps1`) - -Designed specifically for Kubernetes clusters (Kind or Docker Desktop) running the `app-local` and `simplens-infra` pods. - -Instead of crashing Docker directly, it: -1. Generates active background notification traffic using `load-test.js`. -2. Kills specific container processes inside pods via `kubectl exec ... kill 1`. -3. Evicts whole pods via `kubectl delete pod --now`. -4. Disrupts infrastructure (Redis, Kafka) inside the infra pod. -5. Verifies self-healing and outbox recovery. - -#### Usage - -```powershell -# Run all 4 chaos waves with 300 requests: -./scripts/crash-tests/k8s-chaos.ps1 - -# Custom request volume: -./scripts/crash-tests/k8s-chaos.ps1 -Requests 500 - -# Target a specific wave: -./scripts/crash-tests/k8s-chaos.ps1 -Wave app # Individual container kills (worker, processor, delayed) -./scripts/crash-tests/k8s-chaos.ps1 -Wave infra # Redis & Kafka disruptions -./scripts/crash-tests/k8s-chaos.ps1 -Wave pod # Full app-local pod eviction -./scripts/crash-tests/k8s-chaos.ps1 -Wave cascade # Simultaneous app + infra collapse -``` - ---- - -### B. Docker Compose Chaos (`crash-multiwave-all.sh`) - -Designed for environments running directly via `docker compose -f docker-compose.dev.yaml up -d`. - -Simulates 8 distinct disaster waves: -1. **Sustained Load Crash**: Sequential app + Redis kills under heavy load. -2. **Rapid Restart Chaos**: 5 quick crash-restart cycles testing connection pools. -3. **Split Brain Attack**: Partial service availability testing orphan recovery. -4. **Total Blackout**: Complete simultaneous death of app + infra services. -5. **Rolling Chaos**: Continuous random service kills over 60 seconds. -6. **Infrastructure Chaos**: Targeted Redis, Kafka, and MongoDB crashes. -7. **Database Partition**: MongoDB replica crashes during active transactions. -8. **Complete Apocalypse**: Everything killed at once, random recovery race. - -#### Usage (Git Bash / Linux / macOS) - -```bash -cd scripts/crash-tests - -# Run all 8 waves with 500 requests per wave and 30s pause: -./crash-multiwave-all.sh 500 30 -``` - ---- - -## 4. End-to-End Verification Workflow - -To test the entire system end-to-end against a running cluster: - -### Step 1: Start the Webhook Server -In Terminal 1: -```powershell -node scripts/client.js -``` - -### Step 2: Ensure Port Forwarding or NodePorts are Active -In Terminal 2: -```powershell -./k8s/scripts/win/port-forward.ps1 -``` -*(Or use direct Kind NodePorts `http://localhost:30300` for API and `http://localhost:30302` for Dashboard).* - -### Step 3: Run the Chaos Test -In Terminal 3: -```powershell -./scripts/crash-tests/k8s-chaos.ps1 -Requests 300 -``` - -### Step 4: Verify Delivery -- In Terminal 1, watch the incoming webhook deliveries logged. -- Check stats: `curl http://localhost:4000/stats` -- Check Dashboard Alerts: [http://localhost:3002](http://localhost:3002) -- Check Recovery Service Logs: - ```powershell - kubectl logs -n simplens deploy/app-local -c recovery -f - ``` - +--- \ No newline at end of file diff --git a/scripts/crash-tests/crash-multiwave-all.sh b/scripts/crash-tests/crash-multiwave-all.sh deleted file mode 100644 index 341d123..0000000 --- a/scripts/crash-tests/crash-multiwave-all.sh +++ /dev/null @@ -1,843 +0,0 @@ -#!/bin/bash -# ============================================================================== -# Multi-Wave Chaos Crash Test - COMPLETE INFRASTRUCTURE EDITION -# ============================================================================== -# Simulates multiple aggressive waves of crashes on ALL services including -# infrastructure (MongoDB, Redis, Kafka) to maximally stress the notification -# system's recovery mechanisms. -# -# Wave Types: -# 1. Sustained Load Wave - Crashes app services under constant high load -# 2. Rapid Restart Wave - Quick crash-restart cycles (app + infra) -# 3. Split Brain Wave - Processors up/down in patterns -# 4. Total Blackout Wave - EVERYTHING down (incl. infra), staggered recovery -# 5. Rolling Chaos Wave - Random service kills with load -# 6. Infrastructure Chaos Wave - Database, Redis, Kafka crashes -# 7. Database Partition Wave - MongoDB crash and recovery patterns -# 8. Complete Apocalypse Wave - Everything simultaneously, recovery race -# -# Usage: ./crash-multiwave-all.sh [load_requests] [pause_between_waves] -# Example: ./crash-multiwave-all.sh 500 30 -# ============================================================================== - -set -e - -# Configuration -LOAD_REQUESTS=${1:-500} -WAVE_PAUSE=${2:-30} -SCRIPTS_DIR=$(cd "$(dirname "$0")/.." && pwd) -PROJECT_ROOT=$(cd "$(dirname "$0")/../.." && pwd) - -# Determine active compose file -COMPOSE_FILE="$PROJECT_ROOT/docker-compose.dev.yaml" -if [ ! -f "$COMPOSE_FILE" ]; then - COMPOSE_FILE="$PROJECT_ROOT/docker-compose.yaml" -fi - -# Application and Infrastructure Services -# Recovery service is intentionally excluded from kills to keep it monitoring and recovering -APP_SERVICES=( - "worker" - "notification-processor" - "delayed-processor" -) - -INFRA_SERVICES=( - "mongo" - "redis" - "kafka" -) - -# Service identifiers used across chaos waves -APP_CONTAINERS=("${APP_SERVICES[@]}") -INFRA_CONTAINERS=("${INFRA_SERVICES[@]}") -CONTAINERS=("${APP_CONTAINERS[@]}" "${INFRA_CONTAINERS[@]}") -SERVICES=("${APP_SERVICES[@]}" "${INFRA_SERVICES[@]}") - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -MAGENTA='\033[0;35m' -CYAN='\033[0;36m' -NC='\033[0m' # No Color - -# Helper functions -print_header() { - echo "" - echo -e "${MAGENTA}════════════════════════════════════════════════════════════════${NC}" - echo -e "${MAGENTA} $1${NC}" - echo -e "${MAGENTA}════════════════════════════════════════════════════════════════${NC}" - echo "" -} - -print_wave() { - echo "" - echo -e "${CYAN}┌──────────────────────────────────────────────────────────────┐${NC}" - echo -e "${CYAN}│ 🌊 WAVE $1: $2${NC}" - echo -e "${CYAN}└──────────────────────────────────────────────────────────────┘${NC}" - echo "" -} - -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $1" -} - -log_crash() { - echo -e "${RED}[CRASH]${NC} 💥 $1" -} - -log_recover() { - echo -e "${GREEN}[RECOVER]${NC} ✅ $1" -} - -# Kill a service or container cleanly -kill_container() { - local target="$1" - # Try docker compose kill by service name first - if docker compose -f "$COMPOSE_FILE" kill "$target" 2>/dev/null; then - return 0 - fi - # Fallback to direct container kill by name/id - docker kill "$target" 2>/dev/null || true -} - -# Kill multiple containers/services -kill_containers() { - for container in "$@"; do - kill_container "$container" - log_crash "$container" - done -} - -# Restart all application services -restart_all_app_services() { - log_recover "Restarting all application services..." - docker compose -f "$COMPOSE_FILE" up -d "${APP_SERVICES[@]}" - sleep 3 -} - -# Restart all infrastructure services -restart_all_infra_services() { - log_recover "Restarting all infrastructure services..." - docker compose -f "$COMPOSE_FILE" up -d "${INFRA_SERVICES[@]}" - sleep 5 - wait_for_infra_health -} - -# Restart all services (infra first, then app) -restart_all_services() { - log_recover "Restarting all services..." - docker compose -f "$COMPOSE_FILE" up -d "${INFRA_SERVICES[@]}" - sleep 5 - wait_for_infra_health - docker compose -f "$COMPOSE_FILE" up -d "${APP_SERVICES[@]}" - sleep 3 -} - -# Restart specific services -restart_services() { - for service in "$@"; do - log_recover "$service" - docker compose -f "$COMPOSE_FILE" up -d "$service" - done -} - -# Wait for infrastructure to be healthy -wait_for_infra_health() { - log_info "Waiting for infrastructure services to be healthy..." - local max_wait=60 - local waited=0 - - while [ $waited -lt $max_wait ]; do - # Check each service using exit-code based checks - local mongo_ok=0 - local redis_ok=0 - local kafka_ok=0 - - # MongoDB check (try compose exec first, fallback to docker exec) - if docker compose -f "$COMPOSE_FILE" exec -T mongo mongosh --eval "db.adminCommand('ping')" >/dev/null 2>&1 || docker exec mongo mongosh --eval "db.adminCommand('ping')" >/dev/null 2>&1; then - mongo_ok=1 - fi - - # Redis check - if docker compose -f "$COMPOSE_FILE" exec -T redis redis-cli ping 2>/dev/null | grep -q "PONG" || docker exec redis redis-cli ping 2>/dev/null | grep -q "PONG"; then - redis_ok=1 - fi - - # Kafka check - if docker compose -f "$COMPOSE_FILE" exec -T kafka kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1 || docker exec kafka kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1; then - kafka_ok=1 - fi - - if [ $mongo_ok -eq 1 ] && [ $redis_ok -eq 1 ] && [ $kafka_ok -eq 1 ]; then - log_info "All infrastructure services healthy!" - return 0 - fi - - sleep 2 - waited=$((waited + 2)) - done - - log_warn "Timeout waiting for infrastructure health, continuing anyway..." -} - -# Start load test in background -start_load() { - local requests=$1 - log_info "Starting load test with $requests requests..." - cd "$PROJECT_ROOT" - node scripts/load-test.js -n "$requests" -c 50 -ch mock -p mock -m "Chaos test message" -h docker & - LOAD_PID=$! - sleep 2 # Give load test time to start -} - -# Wait for load to finish -wait_for_load() { - if [ -n "$LOAD_PID" ]; then - log_info "Waiting for load test to complete..." - wait $LOAD_PID 2>/dev/null || true - fi -} - -# Random delay between min and max seconds -random_delay() { - local min=$1 - local max=$2 - local delay=$((RANDOM % (max - min + 1) + min)) - sleep $delay -} - -# Get random container(s) from all containers -random_containers() { - local count=$1 - local shuffled=($(shuf -e "${CONTAINERS[@]}")) - echo "${shuffled[@]:0:$count}" -} - -# Get random app container(s) -random_app_containers() { - local count=$1 - local shuffled=($(shuf -e "${APP_CONTAINERS[@]}")) - echo "${shuffled[@]:0:$count}" -} - -# Get random infra container(s) -random_infra_containers() { - local count=$1 - local shuffled=($(shuf -e "${INFRA_CONTAINERS[@]}")) - echo "${shuffled[@]:0:$count}" -} - -# ============================================================================== -# WAVE 1: Sustained Load Crash -# Crashes services while under constant high load -# ============================================================================== -wave_sustained_load() { - print_wave "1/8" "SUSTAINED LOAD CRASH" - log_info "Strategy: Crash app services + Redis sequentially while under constant load" - log_info "This tests message loss and recovery under production-like conditions" - echo "" - - # Start sustained load - start_load $LOAD_REQUESTS - - # Crash services one by one with small delays - sleep 5 - log_crash "Killing notification-processor mid-processing..." - kill_container "${APP_CONTAINERS[1]}" - - sleep 3 - log_crash "Killing worker mid-outbox-poll..." - kill_container "${APP_CONTAINERS[0]}" - - sleep 3 - log_crash "Killing Redis - cache disruption..." - kill_container "redis" - - # Let the chaos continue for a bit - sleep 10 - - # Restart in wrong order (processors before worker, but infra first) - log_info "Restarting infrastructure first..." - restart_services "redis" - sleep 5 - wait_for_infra_health - - log_info "Restarting in wrong order to create backpressure..." - restart_services "notification-processor" - sleep 5 - restart_services "worker" - - wait_for_load - log_info "Wave 1 complete - allowing recovery service to detect issues..." - sleep 15 -} - -# ============================================================================== -# WAVE 2: Rapid Restart Chaos -# Quick crash-restart cycles to stress connection handling -# ============================================================================== -wave_rapid_restart() { - print_wave "2/8" "RAPID RESTART CHAOS" - log_info "Strategy: Rapid crash-restart cycles to stress connection pools" - log_info "This tests Kafka consumer group rebalancing and Redis reconnection" - echo "" - - local cycles=5 - - for i in $(seq 1 $cycles); do - echo "" - log_info "Rapid cycle $i/$cycles" - - # Crash random service - local container=$(random_containers 1) - log_crash "$container" - kill_container "$container" - - # Very short delay - sleep 2 - - # Crash another - container=$(random_containers 1) - log_crash "$container" - kill_container "$container" - - # Quick restart - sleep 3 - restart_all_services - - # Tiny stabilization - sleep 2 - done - - log_info "Wave 2 complete - services may be in unstable state..." - sleep 10 -} - -# ============================================================================== -# WAVE 3: Split Brain Attack -# Create scenarios where some processors are up and others down -# ============================================================================== -wave_split_brain() { - print_wave "3/8" "SPLIT BRAIN ATTACK" - log_info "Strategy: Create inconsistent states with partial service availability" - log_info "This tests status reconciliation and orphaned message detection" - echo "" - - # Start load - start_load $((LOAD_REQUESTS / 2)) - - # Pattern 1: Only notification processor works - log_info "Pattern 1: Only notification processor alive..." - kill_containers "${APP_CONTAINERS[0]}" "${APP_CONTAINERS[2]}" - sleep 15 - restart_all_app_services - sleep 5 - - # Pattern 2: Only delayed processor works - log_info "Pattern 2: Only delayed processor alive..." - kill_containers "${APP_CONTAINERS[0]}" "${APP_CONTAINERS[1]}" - sleep 15 - restart_all_app_services - sleep 5 - - # Pattern 3: Worker down, all processors up (messages accepted but not routed) - log_info "Pattern 3: Worker down - messages queued but not routed..." - kill_container "${APP_CONTAINERS[0]}" - sleep 20 - restart_services "worker" - - wait_for_load - log_info "Wave 3 complete - checking for split brain artifacts..." - sleep 15 -} - -# ============================================================================== -# WAVE 4: Total Blackout -# Everything dies (including infra), then comes back in various orders -# ============================================================================== -wave_total_blackout() { - print_wave "4/8" "TOTAL BLACKOUT" - log_info "Strategy: Complete infrastructure failure including MongoDB, Redis, Kafka" - log_info "This tests system boot order dependencies and data persistence" - echo "" - - # Start load first - start_load $((LOAD_REQUESTS / 3)) - sleep 5 - - # BLACKOUT! - echo "" - echo -e "${RED}╔════════════════════════════════════════════════════════════════╗${NC}" - echo -e "${RED}║ ⚡ TOTAL INFRASTRUCTURE BLACKOUT INITIATED ⚡ ║${NC}" - echo -e "${RED}║ Including: MongoDB, Redis, Kafka, All Processors ║${NC}" - echo -e "${RED}╚════════════════════════════════════════════════════════════════╝${NC}" - echo "" - - # Kill app services first - log_crash "Killing all application services..." - kill_containers "${APP_CONTAINERS[@]}" - - sleep 5 - - # Then infrastructure - this is brutal - log_crash "Killing Redis..." - kill_container "redis" - - log_crash "Killing Kafka..." - kill_container "kafka" - - log_crash "Killing MongoDB..." - kill_container "mongo" - - # Let everything be dead - log_info "Complete infrastructure blackout - everything is dead..." - sleep 20 - - wait_for_load - - # Recovery in proper order - infrastructure first - echo "" - log_info "Beginning staged recovery..." - - log_info "Stage 1: Bringing MongoDB online first (data layer)..." - restart_services "mongo" - sleep 10 - - log_info "Stage 2: Bringing Redis online (cache layer)..." - restart_services "redis" - sleep 5 - - log_info "Stage 3: Bringing Kafka online (message bus)..." - restart_services "kafka" - sleep 10 - - # Wait for infrastructure health - wait_for_infra_health - - log_info "Stage 4: Bringing delayed processor online..." - restart_services "delayed-processor" - sleep 5 - - log_info "Stage 5: Bringing worker online..." - restart_services "worker" - sleep 5 - - log_info "Stage 6: Bringing notification processor online..." - restart_services "notification-processor" - - log_info "Wave 4 complete - system recovering from total blackout..." - sleep 20 -} - -# ============================================================================== -# WAVE 5: Rolling Chaos -# Continuous random crashes including infrastructure with concurrent load -# ============================================================================== -wave_rolling_chaos() { - print_wave "5/8" "ROLLING CHAOS" - log_info "Strategy: Unpredictable random crashes including infrastructure" - log_info "This is a major stress test - pure chaos across all services!" - echo "" - - # Start continuous load - start_load $LOAD_REQUESTS - - local duration=60 # seconds of chaos - local start_time=$(date +%s) - local crash_count=0 - - log_info "Running $duration seconds of pure chaos..." - echo "" - - while true; do - local current_time=$(date +%s) - local elapsed=$((current_time - start_time)) - - if [ $elapsed -ge $duration ]; then - break - fi - - # Random action - expanded to include infra - local action=$((RANDOM % 6)) - - case $action in - 0) - # Kill random app service - local container=$(random_app_containers 1) - log_crash "$container (random app kill)" - kill_container "$container" - ;; - 1) - # Kill two random app services - local containers=($(random_app_containers 2)) - for c in "${containers[@]}"; do - log_crash "$c (double app kill)" - kill_container "$c" - done - ;; - 2) - # Kill all app services then restart one - log_crash "ALL APP SERVICES (chaos burst)" - kill_containers "${APP_CONTAINERS[@]}" - sleep 2 - local service=${APP_SERVICES[$((RANDOM % ${#APP_SERVICES[@]}))]} - log_recover "$service (single survivor)" - restart_services "$service" - ;; - 3) - # Quick restart cycle - local container=$(random_app_containers 1) - log_crash "$container (restart cycle)" - kill_container "$container" - sleep 1 - restart_all_app_services - ;; - 4) - # Kill Redis briefly - log_crash "redis (brief infra kill)" - kill_container "redis" - sleep 3 - restart_services "redis" - ;; - 5) - # Kill random infrastructure service - local infra_container=$(random_infra_containers 1) - log_crash "$infra_container (infra chaos)" - kill_container "$infra_container" - sleep 5 - restart_all_infra_services - ;; - esac - - crash_count=$((crash_count + 1)) - random_delay 3 8 - done - - echo "" - log_info "Chaos complete! Total crash events: $crash_count" - - # Final recovery - restart_all_services - wait_for_load - - log_info "Wave 5 complete - final recovery initiated..." - sleep 15 -} - -# ============================================================================== -# WAVE 6: Infrastructure Chaos -# Specifically targets MongoDB, Redis, and Kafka -# ============================================================================== -wave_infrastructure_chaos() { - print_wave "6/8" "INFRASTRUCTURE CHAOS" - log_info "Strategy: Targeted attacks on core infrastructure components" - log_info "This tests data layer resilience and connection recovery" - echo "" - - # Pattern 1: Redis crash while processing - log_info "Pattern 1: Redis crash during active processing..." - start_load $((LOAD_REQUESTS / 4)) - sleep 5 - log_crash "Killing Redis..." - kill_container "redis" - sleep 15 - log_recover "Restarting Redis..." - restart_services "redis" - wait_for_load - sleep 10 - - # Pattern 2: Kafka crash - message broker down - log_info "Pattern 2: Kafka crash - message broker failure..." - start_load $((LOAD_REQUESTS / 4)) - sleep 5 - log_crash "Killing Kafka..." - kill_container "kafka" - sleep 20 # Kafka takes longer to recover - log_recover "Restarting Kafka..." - restart_services "kafka" - sleep 15 - wait_for_load - sleep 10 - - # Pattern 3: MongoDB crash - primary database failure - log_info "Pattern 3: MongoDB crash - database failure..." - start_load $((LOAD_REQUESTS / 4)) - sleep 5 - log_crash "Killing MongoDB..." - kill_container "mongo" - sleep 20 - log_recover "Restarting MongoDB..." - restart_services "mongo" - sleep 15 - wait_for_infra_health - wait_for_load - sleep 10 - - # Pattern 4: Cascading infrastructure failure - log_info "Pattern 4: Cascading infrastructure failure..." - start_load $((LOAD_REQUESTS / 4)) - sleep 5 - - log_crash "Redis down..." - kill_container "redis" - sleep 3 - log_crash "Kafka down..." - kill_container "kafka" - sleep 15 - - # Recovery in reverse order - log_recover "Kafka recovering..." - restart_services "kafka" - sleep 5 - log_recover "Redis recovering..." - restart_services "redis" - - wait_for_infra_health - wait_for_load - - log_info "Wave 6 complete - infrastructure chaos survived!" - sleep 15 -} - -# ============================================================================== -# WAVE 7: Database Partition -# MongoDB specific crashes to test data consistency -# ============================================================================== -wave_database_partition() { - print_wave "7/8" "DATABASE PARTITION" - log_info "Strategy: MongoDB crash patterns to test data consistency" - log_info "This tests transaction handling and outbox pattern resilience" - echo "" - - # Pattern 1: DB crash during outbox poll - log_info "Pattern 1: MongoDB crash during outbox poll..." - start_load $((LOAD_REQUESTS / 3)) - sleep 10 # Let messages start processing - - log_crash "MongoDB crash mid-transaction..." - kill_container "mongo" - - # Workers should be stuck, messages should be in various states - sleep 20 - - log_recover "MongoDB recovering..." - restart_services "mongo" - sleep 10 - wait_for_infra_health - wait_for_load - sleep 10 - - # Pattern 2: Rapid MongoDB restart - log_info "Pattern 2: Rapid MongoDB restart cycles..." - for i in $(seq 1 3); do - log_info "MongoDB restart cycle $i/3" - log_crash "MongoDB down..." - kill_container "mongo" - sleep 5 - log_recover "MongoDB up..." - restart_services "mongo" - sleep 10 - done - wait_for_infra_health - sleep 10 - - # Pattern 3: DB + App service coordinated crash - log_info "Pattern 3: Coordinated DB and worker crash..." - start_load $((LOAD_REQUESTS / 3)) - sleep 5 - - log_crash "Worker + MongoDB simultaneous crash..." - kill_container "${APP_CONTAINERS[0]}" - kill_container "mongo" - sleep 15 - - log_recover "MongoDB first, then worker..." - restart_services "mongo" - sleep 10 - wait_for_infra_health - restart_services "worker" - - wait_for_load - log_info "Wave 7 complete - database partition test finished!" - sleep 15 -} - -# ============================================================================== -# WAVE 8: Complete Apocalypse -# Everything crashes simultaneously, system races to recover -# ============================================================================== -wave_complete_apocalypse() { - print_wave "8/8" "COMPLETE APOCALYPSE" - log_info "Strategy: Simultaneous crash of EVERYTHING - the ultimate test" - log_info "This tests total system recovery from catastrophic failure" - echo "" - - # Start heavy load - start_load $LOAD_REQUESTS - sleep 10 # Let processing begin - - echo "" - echo -e "${RED}╔════════════════════════════════════════════════════════════════╗${NC}" - echo -e "${RED}║ ║${NC}" - echo -e "${RED}║ 💀 COMPLETE APOCALYPSE - KILLING EVERYTHING 💀 ║${NC}" - echo -e "${RED}║ ║${NC}" - echo -e "${RED}║ MongoDB 🔴 Redis 🔴 Kafka 🔴 ║${NC}" - echo -e "${RED}║ Worker 🔴 Notification 🔴 Delayed 🔴 ║${NC}" - echo -e "${RED}║ ║${NC}" - echo -e "${RED}╚════════════════════════════════════════════════════════════════╝${NC}" - echo "" - - # Kill EVERYTHING at once - log_crash "Killing ALL services simultaneously..." - for container in "${CONTAINERS[@]}"; do - kill_container "$container" & - done - wait # Wait for all kills to complete - - # Total darkness - log_info "Complete system failure - all services dead..." - sleep 30 - - wait_for_load - - # Chaotic recovery - random order - echo "" - log_info "Beginning chaotic recovery..." - - # Shuffle and recover in random order (but infrastructure first) - log_info "Recovering infrastructure in random order..." - local shuffled_infra=($(shuf -e "${INFRA_SERVICES[@]}")) - for service in "${shuffled_infra[@]}"; do - log_recover "$service (random infra recovery)" - restart_services "$service" - sleep 5 - done - - wait_for_infra_health - - log_info "Recovering app services in random order..." - local shuffled_app=($(shuf -e "${APP_SERVICES[@]}")) - for service in "${shuffled_app[@]}"; do - log_recover "$service (random app recovery)" - restart_services "$service" - sleep 3 - done - - log_info "Wave 8 complete - survived the apocalypse!" - sleep 20 -} - -# ============================================================================== -# Main Execution -# ============================================================================== -main() { - print_header "🔥 MULTI-WAVE CHAOS CRASH TEST - COMPLETE INFRASTRUCTURE EDITION 🔥" - - echo "Configuration:" - echo " Load Requests per Wave: $LOAD_REQUESTS" - echo " Pause Between Waves: ${WAVE_PAUSE}s" - echo " Project Root: $PROJECT_ROOT" - echo "" - echo "Services to be crashed:" - echo " Application: worker, notification-processor (unified), delayed-processor" - echo " Infrastructure: MongoDB, Redis, Kafka" - echo "" - - log_warn "⚠️ This test WILL crash ALL your services including infrastructure!" - log_warn "⚠️ MongoDB, Redis, and Kafka WILL be killed during this test!" - log_warn "Make sure recovery service is running: docker logs -f ns-recovery" - echo "" - log_info "Starting in 5 seconds... (Ctrl+C to abort)" - sleep 5 - - # Ensure all services are running first - log_info "Ensuring all services are running..." - restart_all_services - sleep 5 - - # Execute waves - wave_sustained_load - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_rapid_restart - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_split_brain - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_total_blackout - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_rolling_chaos - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_infrastructure_chaos - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_database_partition - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before final wave..." - sleep $WAVE_PAUSE - - wave_complete_apocalypse - - # Final summary - print_header "🏁 COMPLETE INFRASTRUCTURE CHAOS TEST COMPLETE 🏁" - - echo "All 8 waves completed! Your system survived:" - echo " ✓ Application service crashes" - echo " ✓ Infrastructure crashes (MongoDB, Redis, Kafka)" - echo " ✓ Complete blackouts" - echo " ✓ Database partitions" - echo " ✓ The apocalypse" - echo "" - echo "Review the following:" - echo "" - echo "1. Recovery Service Logs:" - echo " docker compose -f \"$COMPOSE_FILE\" logs -f recovery" - echo "" - echo "2. Dashboard Alerts:" - echo " http://localhost:3002" - echo "" - echo "3. Check for stuck notifications:" - echo " docker compose -f \"$COMPOSE_FILE\" exec -T mongo mongosh --eval \"use notification_service; db.notifications.countDocuments({status: 'processing'})\"" - echo "" - echo "4. Check Kafka consumer lag:" - echo " docker compose -f \"$COMPOSE_FILE\" exec -T kafka kafka-broker-api-versions.sh --bootstrap-server localhost:9092" - echo "" - echo "5. Check Redis connectivity:" - echo " docker compose -f \"$COMPOSE_FILE\" exec -T redis redis-cli ping" - echo "" - echo "6. Check MongoDB replica set status:" - echo " docker compose -f \"$COMPOSE_FILE\" exec -T mongo mongosh --eval \"rs.status()\"" - echo "" - - log_info "Final service restart..." - restart_all_services - - echo "" - log_info "✅ All services restored. Monitor recovery for the next few minutes." - log_info "🎉 Congratulations! Your system survived complete infrastructure chaos!" -} - -# Run main -main "$@" diff --git a/scripts/crash-tests/k8s-chaos.ps1 b/scripts/crash-tests/k8s-chaos.ps1 deleted file mode 100644 index 39aac59..0000000 --- a/scripts/crash-tests/k8s-chaos.ps1 +++ /dev/null @@ -1,159 +0,0 @@ -<# -.SYNOPSIS - Kubernetes Chaos & Crash Test Script for SimpleNS - Simulates container failures, pod evictions, and infrastructure disruptions - against running Kubernetes pods (app-local and simplens-infra). - -.PARAMETER Requests - Number of background test requests to send during each chaos wave (default: 300). - -.PARAMETER Wave - Specific wave to run: 'all', 'app', 'infra', 'pod', 'cascade' (default: 'all'). -#> -param( - [int]$Requests = 300, - [ValidateSet("all", "app", "infra", "pod", "cascade")] - [string]$Wave = "all" -) - -$ErrorActionPreference = "Continue" -$Namespace = "simplens" -$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path -$RootDir = Split-Path -Parent (Split-Path -Parent $ScriptDir) - -function Log-Info($msg) { Write-Host "[INFO] $msg" -ForegroundColor Cyan } -function Log-Crash($msg) { Write-Host "[CRASH] $msg" -ForegroundColor Red } -function Log-Recover($msg) { Write-Host "[RECOVER] $msg" -ForegroundColor Green } -function Log-Wave($num, $title) { - Write-Host "`n================================================================" -ForegroundColor Magenta - Write-Host " [WAVE $num] $title" -ForegroundColor Magenta - Write-Host "================================================================`n" -ForegroundColor Magenta -} - -# Start background load using load-test.js -function Start-BackgroundLoad([int]$count) { - Log-Info "Starting background load ($count requests) against API..." - $job = Start-Job -ScriptBlock { - param($root, $n) - Set-Location $root - node "$root/scripts/load-test.js" -n $n -c 25 -ch mock -p mock -h docker - } -ArgumentList $RootDir, $count - Start-Sleep -Seconds 2 - return $job -} - -# 1. Kill a specific container inside app-local pod -function Kill-AppContainer([string]$containerName) { - Log-Crash "Killing container '$containerName' in pod 'app-local'..." - kubectl exec -n $Namespace deploy/app-local -c $containerName -- kill 1 2>$null -} - -# 2. Kill a specific container inside simplens-infra pod -function Kill-InfraContainer([string]$containerName) { - Log-Crash "Killing container '$containerName' in pod 'simplens-infra'..." - if ($containerName -eq "redis") { - kubectl exec -n $Namespace deploy/simplens-infra -c redis -- redis-cli shutdown nosave 2>$null - } elseif ($containerName -eq "mongo") { - kubectl exec -n $Namespace deploy/simplens-infra -c mongo -- mongosh --eval "db.adminCommand({shutdown: 1, force: true})" 2>$null - } else { - kubectl exec -n $Namespace deploy/simplens-infra -c $containerName -- kill 1 2>$null - } -} - -# 3. Wait for app pod ready -function Wait-AppReady { - Log-Info "Waiting for app-local pod to return to Ready state..." - kubectl wait --namespace $Namespace --for=condition=ready pod -l app.kubernetes.io/name=app-local --timeout=60s 2>$null -} - -# 4. Wait for infra pod ready -function Wait-InfraReady { - Log-Info "Waiting for simplens-infra pod to return to Ready state..." - kubectl wait --namespace $Namespace --for=condition=ready pod -l app.kubernetes.io/name=simplens-infra --timeout=90s 2>$null -} - -Write-Host "================================================================" -ForegroundColor Yellow -Write-Host " SimpleNS Kubernetes Native Chaos & Resilience Test" -ForegroundColor Yellow -Write-Host " Target Cluster: Kind | Namespace: $Namespace" -ForegroundColor Yellow -Write-Host "================================================================" -ForegroundColor Yellow - -# Wave 1: Application Container Failures -if ($Wave -eq "all" -or $Wave -eq "app") { - Log-Wave "1" "INDIVIDUAL APP CONTAINER CRASHES UNDER LOAD" - $loadJob = Start-BackgroundLoad $Requests - - Start-Sleep -Seconds 3 - Kill-AppContainer "notification-processor" - Start-Sleep -Seconds 5 - - Kill-AppContainer "worker" - Start-Sleep -Seconds 5 - - Kill-AppContainer "delayed-processor" - - Log-Info "Waiting for containers to self-heal and load to complete..." - Wait-AppReady - $loadJob | Wait-Job | Receive-Job - $loadJob | Remove-Job -Force -} - -# Wave 2: Infrastructure Disruptions (Kafka / Redis) -if ($Wave -eq "all" -or $Wave -eq "infra") { - Log-Wave "2" "INFRASTRUCTURE DISRUPTIONS (Redis & Kafka)" - $loadJob = Start-BackgroundLoad $Requests - - Start-Sleep -Seconds 3 - Kill-InfraContainer "redis" - Start-Sleep -Seconds 6 - - Kill-InfraContainer "kafka" - Start-Sleep -Seconds 8 - - Wait-InfraReady - Wait-AppReady - $loadJob | Wait-Job | Receive-Job - $loadJob | Remove-Job -Force -} - -# Wave 3: Entire Pod Eviction / Hard Kill -if ($Wave -eq "all" -or $Wave -eq "pod") { - Log-Wave "3" "FULL APP-LOCAL POD EVICTION" - $loadJob = Start-BackgroundLoad $Requests - - Start-Sleep -Seconds 3 - Log-Crash "Force-deleting the entire 'app-local' pod!" - kubectl delete pod -n $Namespace -l app.kubernetes.io/name=app-local --now 2>$null - - Log-Info "Kubelet is recreating the pod..." - Wait-AppReady - $loadJob | Wait-Job | Receive-Job - $loadJob | Remove-Job -Force -} - -# Wave 4: Cascading Failure -if ($Wave -eq "all" -or $Wave -eq "cascade") { - Log-Wave "4" "CASCADING CHAOS (Infra + App simultaneous death)" - $loadJob = Start-BackgroundLoad $Requests - - Start-Sleep -Seconds 3 - Kill-InfraContainer "redis" - Kill-AppContainer "worker" - Kill-AppContainer "notification-processor" - - Start-Sleep -Seconds 6 - Log-Crash "Evicting app pod while Redis is recovering..." - kubectl delete pod -n $Namespace -l app.kubernetes.io/name=app-local --now 2>$null - - Wait-InfraReady - Wait-AppReady - $loadJob | Wait-Job | Receive-Job - $loadJob | Remove-Job -Force -} - -Write-Host "`n================================================================" -ForegroundColor Green -Write-Host " [OK] Kubernetes Chaos Test Complete!" -ForegroundColor Green -Write-Host "================================================================" -ForegroundColor Green -Write-Host "Verify system status:" -ForegroundColor Cyan -Write-Host " - Pod status: kubectl get pods -n $Namespace" -ForegroundColor White -Write-Host " - Recovery Logs: kubectl logs -n $Namespace deploy/app-local -c recovery" -ForegroundColor White -Write-Host " - Dashboard Alerts: http://localhost:3002" -ForegroundColor White From 3e0873c6743b70a25c6c9edecc9190024ff37cd7 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Tue, 15 Sep 2026 00:07:45 +0530 Subject: [PATCH 03/46] feat: implement centralized plugin management and synchronization across api and notification processor services --- dashboard/app/plugins/page.tsx | 370 ++---------- .../plugins/channel-routing-tab.tsx | 382 ++++++++++++ .../plugins/installed-plugins-tab.tsx | 367 ++++++++++++ .../components/plugins/providers-tab.tsx | 547 ++++++++++++++++++ dashboard/lib/api-client.ts | 54 ++ dashboard/lib/types.ts | 51 ++ .../controllers/channel-routing.controller.ts | 125 ++++ src/api/controllers/plugins.controller.ts | 156 ++++- src/api/controllers/providers.controller.ts | 190 ++++++ src/api/routes/channel-routing.routes.ts | 22 + src/api/routes/plugins.routes.ts | 20 +- src/api/routes/providers.routes.ts | 26 + src/api/server.ts | 37 +- src/config/kafka.config.ts | 99 +++- src/database/models/admin-channel.models.ts | 10 +- src/database/models/channel-routing.models.ts | 45 ++ src/database/models/plugin.models.ts | 61 ++ src/database/models/provider.models.ts | 74 +++ src/database/models/system-config.models.ts | 4 +- src/plugins/bootstrap/yaml-migrator.ts | 251 ++++++++ src/plugins/crypto/keypair-manager.ts | 173 ++++++ src/plugins/index.ts | 23 +- src/plugins/loader/loader.ts | 85 +++ src/plugins/loader/plugin-fs.ts | 191 ++++++ src/plugins/loader/registry.ts | 97 +++- src/plugins/loader/router.ts | 143 +++-- .../services/channel-routing.service.ts | 153 +++++ .../services/plugin-manager.service.ts | 196 +++++++ .../services/provider-manager.service.ts | 365 ++++++++++++ src/plugins/sync/plugin-sync.service.ts | 194 +++++++ src/processors/unified/unified.consumer.ts | 7 + src/processors/unified/unified.processor.ts | 372 +++++++----- src/types/schemas.ts | 67 ++- src/types/types.ts | 16 + .../plugins/channel-routing.service.test.ts | 114 ++++ tests/unit/plugins/crypto.test.ts | 118 ++++ .../plugins/plugin-manager.service.test.ts | 142 +++++ .../plugins/provider-manager.service.test.ts | 147 +++++ tests/unit/plugins/router.test.ts | 1 + tsconfig.json | 1 + 40 files changed, 4925 insertions(+), 571 deletions(-) create mode 100644 dashboard/components/plugins/channel-routing-tab.tsx create mode 100644 dashboard/components/plugins/installed-plugins-tab.tsx create mode 100644 dashboard/components/plugins/providers-tab.tsx create mode 100644 src/api/controllers/channel-routing.controller.ts create mode 100644 src/api/controllers/providers.controller.ts create mode 100644 src/api/routes/channel-routing.routes.ts create mode 100644 src/api/routes/providers.routes.ts create mode 100644 src/database/models/channel-routing.models.ts create mode 100644 src/database/models/plugin.models.ts create mode 100644 src/database/models/provider.models.ts create mode 100644 src/plugins/bootstrap/yaml-migrator.ts create mode 100644 src/plugins/crypto/keypair-manager.ts create mode 100644 src/plugins/loader/plugin-fs.ts create mode 100644 src/plugins/services/channel-routing.service.ts create mode 100644 src/plugins/services/plugin-manager.service.ts create mode 100644 src/plugins/services/provider-manager.service.ts create mode 100644 src/plugins/sync/plugin-sync.service.ts create mode 100644 tests/unit/plugins/channel-routing.service.test.ts create mode 100644 tests/unit/plugins/crypto.test.ts create mode 100644 tests/unit/plugins/plugin-manager.service.test.ts create mode 100644 tests/unit/plugins/provider-manager.service.test.ts diff --git a/dashboard/app/plugins/page.tsx b/dashboard/app/plugins/page.tsx index decf9d4..d3b835b 100644 --- a/dashboard/app/plugins/page.tsx +++ b/dashboard/app/plugins/page.tsx @@ -1,315 +1,71 @@ "use client"; +import { useState } from "react"; import { DashboardLayout } from "@/components/layout/dashboard-layout"; import { PageToolbar, PageToolbarSection, PageToolbarSpacer } from "@/components/ui/page-toolbar"; -import { Card, CardContent } from "@/components/ui/card"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { Skeleton } from "@/components/ui/skeleton"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import useSWR from "swr"; -import { Puzzle, RefreshCw, Mail, MessageCircle, TestTube2, Zap, User, FileText } from "lucide-react"; -import { useMemo, useState } from "react"; -import { apiClient } from "@/lib/api-client"; -import { ProviderMetadata, PluginMetadata as PluginsResponse } from "@/lib/types"; - -const fetcher = (url: string): Promise => apiClient.get(url) as unknown as Promise; - -// Channel-specific colors and icons -const channelConfig: Record; color: string }> = { - email: { - icon: Mail, - color: "text-blue-500", - }, - whatsapp: { - icon: MessageCircle, - color: "text-green-500", - }, - mock: { - icon: TestTube2, - color: "text-purple-500", - }, - sms: { - icon: MessageCircle, - color: "text-orange-500", - }, -}; - -const getChannelConfig = (channel: string) => { - return channelConfig[channel] || { - icon: Zap, - color: "text-green-500", - }; -}; - -interface PluginCardProps { - provider: ProviderMetadata; - channel: string; - isDefault: boolean; -} - -function PluginCard({ provider, channel, isDefault }: PluginCardProps) { - const config = getChannelConfig(channel); - const ChannelIcon = config.icon; - - return ( - - {/* Default Badge */} - {isDefault && ( -
- - Default - -
- )} - -
- {/* Header */} -
-
- -
-
-

- {provider.displayName || provider.name} -

-

- ID: {provider.id} -

-
-
- - {/* Channel & Priority */} -
-
- - Channel: - {channel} -
-
- Priority: - {provider.priority} -
-
- - {/* Recipient Fields */} - {provider.recipientFields?.length > 0 && ( -
-
- - Recipient Fields: -
-
- {provider.recipientFields.map((field) => ( - - {field.name}{field.required && "*"} - - ))} -
-
- )} - - {/* Content Fields */} - {provider.contentFields?.length > 0 && ( -
-
- - Content Fields: -
-
- {provider.contentFields.map((field) => ( - - {field.name}{field.required && "*"} - - ))} -
-
- )} - - {/* Description */} - {provider.description && ( -

- {provider.description} -

- )} -
- - ); -} - -function PluginCardSkeleton() { - return ( - -
-
- -
- - -
- -
-
- - ); -} +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Puzzle, Server, Package, GitBranch } from "lucide-react"; +import { ProvidersTab } from "@/components/plugins/providers-tab"; +import { InstalledPluginsTab } from "@/components/plugins/installed-plugins-tab"; +import { ChannelRoutingTab } from "@/components/plugins/channel-routing-tab"; export default function PluginsPage() { - const [channelFilter, setChannelFilter] = useState("all"); - - const { data, isLoading, error, mutate } = useSWR( - "/api/plugins", - fetcher, - { refreshInterval: 30000 } - ); - - // Flatten channels into a list of providers with channel info - const allProviders = useMemo(() => { - if (!data?.channels) return []; - - const providers: Array<{ provider: ProviderMetadata; channel: string; isDefault: boolean }> = []; - - for (const [channel, channelData] of Object.entries(data.channels)) { - for (const provider of channelData.providers) { - providers.push({ - provider, - channel, - isDefault: channelData.default === provider.id - }); - } - } - - return providers; - }, [data]); - - // Get available channels for filter - const availableChannels = useMemo(() => { - if (!data?.channels) return []; - return Object.keys(data.channels); - }, [data]); - - // Filter providers by selected channel - const filteredProviders = useMemo(() => { - if (channelFilter === "all") return allProviders; - return allProviders.filter(p => p.channel === channelFilter); - }, [allProviders, channelFilter]); - - const channelCount = data?.channels ? Object.keys(data.channels).length : 0; - - return ( - -
- {/* Toolbar */} - - -
- - - {isLoading ? ( - - ) : ( - `${allProviders.length} plugin(s) installed • ${channelCount} channel(s) available` - )} - -
- - - - - - - - - - - {/* Loading State */} - {isLoading && ( -
- {[...Array(3)].map((_, i) => ( - - ))} -
- )} - - {/* Error State */} - {error && ( - - -

Failed to load plugins. Make sure the API is running.

- - - )} - - {/* Empty State */} - {data && allProviders.length === 0 && ( - - -
- -
-

No Plugins Installed

-

- Get started by installing your first notification plugin. -

- - - - )} - - {/* Plugin Cards */} - {data && filteredProviders.length > 0 && ( -
- {filteredProviders.map(({ provider, channel, isDefault }) => ( - - ))} -
- )} + const [activeTab, setActiveTab] = useState("providers"); + + return ( + +
+ {/* Page Toolbar */} + + +
+
+ +
+
+

Plugin & Provider Hub

+

+ Dynamically manage provider plugins, encrypted credentials, and cascading routing. +

+
- - ); + + + + + {/* 3-Tab Interface */} + + + + + Providers + + + + Installed Plugins + + + + Channel Routing + + + + + + + + + + + + + + + +
+ + ); } diff --git a/dashboard/components/plugins/channel-routing-tab.tsx b/dashboard/components/plugins/channel-routing-tab.tsx new file mode 100644 index 0000000..88e1915 --- /dev/null +++ b/dashboard/components/plugins/channel-routing-tab.tsx @@ -0,0 +1,382 @@ +"use client"; + +import { useState } from "react"; +import useSWR from "swr"; +import { channelRoutingService, providerService } from "@/lib/api-client"; +import type { ChannelRoutingDto, ProviderDto } from "@/lib/types"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + GitBranch, + RefreshCw, + Edit2, + Plus, + ArrowRight, + Cpu, + Loader2, + AlertCircle, + X, +} from "lucide-react"; +import { toast } from "sonner"; + +export function ChannelRoutingTab() { + const { + data: routingData, + isLoading: routingLoading, + mutate: mutateRouting, + } = useSWR<{ routings: ChannelRoutingDto[] }>("/api/channels/routing", () => + channelRoutingService.list() + ); + + const { data: providersData } = useSWR<{ providers: ProviderDto[] }>("/api/providers", () => + providerService.list() + ); + + const [modalOpen, setModalOpen] = useState(false); + const [editingRouting, setEditingRouting] = useState(null); + + // Form State + const [channel, setChannel] = useState(""); + const [defaultProviderId, setDefaultProviderId] = useState(""); + const [fallbackIds, setFallbackIds] = useState([]); + const [partitions, setPartitions] = useState(6); + const [saving, setSaving] = useState(false); + + const allProviders = providersData?.providers || []; + const routings = routingData?.routings || []; + + // Filter providers by selected channel + const channelProviders = allProviders.filter((p) => !channel || p.channel === channel); + + const handleOpenEdit = (route: ChannelRoutingDto) => { + setEditingRouting(route); + setChannel(route.channel); + setDefaultProviderId(route.default_provider_id); + setFallbackIds(route.fallback_provider_ids || []); + setPartitions(route.partitions || 6); + setModalOpen(true); + }; + + const handleOpenCreate = () => { + setEditingRouting(null); + setChannel(""); + setDefaultProviderId(""); + setFallbackIds([]); + setPartitions(6); + setModalOpen(true); + }; + + const handleAddFallback = (provId: string) => { + if (!provId || provId === defaultProviderId || fallbackIds.includes(provId)) return; + setFallbackIds([...fallbackIds, provId]); + }; + + const handleRemoveFallback = (index: number) => { + setFallbackIds(fallbackIds.filter((_, i) => i !== index)); + }; + + const handleSave = async (e: React.FormEvent) => { + e.preventDefault(); + if (!channel.trim()) { + toast.error("Channel name is required"); + return; + } + if (!defaultProviderId.trim()) { + toast.error("Default primary provider is required"); + return; + } + + setSaving(true); + try { + await channelRoutingService.set(channel.trim(), { + default_provider_id: defaultProviderId.trim(), + fallback_provider_ids: fallbackIds, + partitions, + }); + + toast.success(`Routing for channel '${channel}' updated`); + setModalOpen(false); + mutateRouting(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to update channel routing"; + toast.error(message); + } finally { + setSaving(false); + } + }; + + return ( +
+ {/* Header */} +
+
+

Channel Routing & Kafka Scaling

+

+ Configure primary and cascading fallback providers per channel, and scale topic partitions. +

+
+
+ + +
+
+ + {/* Routing Cards Grid */} + {routingLoading ? ( +
+ {[1, 2].map((i) => ( + + + + + + + + ))} +
+ ) : routings.length === 0 ? ( + + + +
+

No channel routings configured

+

+ Map channels to primary and cascading fallback providers. +

+
+ + + + ) : ( +
+ {routings.map((route) => ( + + +
+
+ + {route.channel} + +
+
+ + {route.partitions || 6} Partitions +
+
+ + {/* Routing Cascade Chain */} +
+
+ Cascading Failover Route +
+
+ + Primary: {route.default_provider_id} + + + {route.fallback_provider_ids && route.fallback_provider_ids.length > 0 ? ( + route.fallback_provider_ids.map((fId, idx) => ( +
+ + + Fallback {idx + 1}: {fId} + +
+ )) + ) : ( + + (No fallbacks configured) + + )} +
+
+ +
+ +
+ + + ))} +
+ )} + + {/* Edit/Configure Routing Dialog */} + + +
+ + {editingRouting ? "Edit Channel Routing" : "Configure Channel"} + + Configure primary provider, ordered fallback failover cascade, and Kafka partition target. + + + +
+
+ + setChannel(e.target.value.toLowerCase())} + disabled={!!editingRouting} + required + /> +
+ +
+ + +
+ + {/* Cascading Fallback Array */} +
+ +

+ If the primary provider fails with a non-retryable error, SimpleNS tries each fallback in order. +

+ + {fallbackIds.length > 0 && ( +
+ {fallbackIds.map((fId, idx) => ( +
+ + {idx + 1}. {fId} + + +
+ ))} +
+ )} + +
+ +
+
+ + {/* Kafka Partitions Scaling */} +
+
+ + + Min: {editingRouting?.partitions || 1} + +
+ setPartitions(parseInt(e.target.value, 10) || 1)} + required + /> +

+ + Partitions can only be scaled up, never decreased. +

+
+
+ + + + + +
+ + +
+ ); +} diff --git a/dashboard/components/plugins/installed-plugins-tab.tsx b/dashboard/components/plugins/installed-plugins-tab.tsx new file mode 100644 index 0000000..de90e2e --- /dev/null +++ b/dashboard/components/plugins/installed-plugins-tab.tsx @@ -0,0 +1,367 @@ +"use client"; + +import { useState } from "react"; +import useSWR from "swr"; +import { pluginService } from "@/lib/api-client"; +import type { InstalledPlugin } from "@/lib/types"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Download, Package, RefreshCw, Trash2, ArrowUpDown, Loader2 } from "lucide-react"; +import { toast } from "sonner"; + +export function InstalledPluginsTab() { + const { data, error, isLoading, mutate } = useSWR<{ plugins: InstalledPlugin[] }>( + "/api/plugins/installed", + () => pluginService.listInstalled() + ); + + const [installModalOpen, setInstallModalOpen] = useState(false); + const [packageName, setPackageName] = useState(""); + const [packageVersion, setPackageVersion] = useState(""); + const [installing, setInstalling] = useState(false); + + const [versionModalOpen, setVersionModalOpen] = useState(false); + const [selectedPlugin, setSelectedPlugin] = useState(null); + const [targetVersion, setTargetVersion] = useState(""); + const [updatingVersion, setUpdatingVersion] = useState(false); + + const [uninstallDialogOpen, setUninstallDialogOpen] = useState(false); + const [pluginToUninstall, setPluginToUninstall] = useState(null); + const [uninstalling, setUninstalling] = useState(false); + + const handleInstall = async (e: React.FormEvent) => { + e.preventDefault(); + if (!packageName.trim()) return; + + setInstalling(true); + try { + await pluginService.install(packageName.trim(), packageVersion.trim() || undefined); + toast.success(`Plugin '${packageName}' installed successfully`); + setInstallModalOpen(false); + setPackageName(""); + setPackageVersion(""); + mutate(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to install plugin"; + toast.error(message); + } finally { + setInstalling(false); + } + }; + + const handleChangeVersion = async (e: React.FormEvent) => { + e.preventDefault(); + if (!selectedPlugin || !targetVersion.trim()) return; + + setUpdatingVersion(true); + try { + await pluginService.changeVersion(selectedPlugin.name, targetVersion.trim()); + toast.success(`Plugin '${selectedPlugin.name}' updated to ${targetVersion.trim()}`); + setVersionModalOpen(false); + setSelectedPlugin(null); + setTargetVersion(""); + mutate(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to change plugin version"; + toast.error(message); + } finally { + setUpdatingVersion(false); + } + }; + + const handleUninstall = async () => { + if (!pluginToUninstall) return; + + setUninstalling(true); + try { + await pluginService.uninstall(pluginToUninstall); + toast.success(`Plugin '${pluginToUninstall}' uninstalled successfully`); + setUninstallDialogOpen(false); + setPluginToUninstall(null); + mutate(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to uninstall plugin"; + toast.error(message); + } finally { + setUninstalling(false); + } + }; + + const plugins = data?.plugins || []; + + return ( +
+ {/* Action Header */} +
+
+

Installed Plugin Packages

+

+ Manage npm provider packages installed into SimpleNS. +

+
+
+ + +
+
+ + {/* Plugins Grid */} + {isLoading ? ( +
+ {[1, 2, 3].map((i) => ( + + + + + + + + ))} +
+ ) : error ? ( + + + Failed to load installed plugins. + + + ) : plugins.length === 0 ? ( + + + +
+

No plugins installed

+

+ Install provider packages such as @simplens/nodemailer-gmail or @simplens/mock. +

+
+ + + + ) : ( +
+ {plugins.map((plugin) => ( + + +
+
+

{plugin.name}

+

+ {plugin.manifest?.displayName || plugin.name} +

+
+ + v{plugin.version} + +
+ +
+ + Channel: {plugin.manifest?.channel || "generic"} + + + {plugin.status} + +
+ + {plugin.manifest?.description && ( +

+ {plugin.manifest.description} +

+ )} + + {plugin.manifest?.requiredCredentials && plugin.manifest.requiredCredentials.length > 0 && ( +
+ Credentials required:{" "} + {plugin.manifest.requiredCredentials.join(", ")} +
+ )} + +
+ + +
+ + + ))} +
+ )} + + {/* Install Plugin Dialog */} + + +
+ + Install Plugin Package + + Install a new notification provider package from npm into SimpleNS. + + +
+
+ + setPackageName(e.target.value)} + required + /> +
+
+ + setPackageVersion(e.target.value)} + /> +
+
+ + + + +
+ + + + {/* Change Version Dialog */} + + +
+ + Change Plugin Version + + Upgrade or downgrade {selectedPlugin?.name} to a specific version. + + +
+
+ + setTargetVersion(e.target.value)} + required + /> +
+
+ + + + +
+ + + + {/* Uninstall Confirmation Dialog */} + + + + Uninstall Plugin + + Are you sure you want to uninstall {pluginToUninstall}? This will delete the package from + .plugins/node_modules. Uninstallation will be blocked if any configured providers still depend on this plugin. + + + + Cancel + { + e.preventDefault(); + handleUninstall(); + }} + disabled={uninstalling} + className="bg-destructive text-destructive-foreground hover:bg-destructive/90" + > + {uninstalling ? "Uninstalling..." : "Uninstall"} + + + + +
+ ); +} diff --git a/dashboard/components/plugins/providers-tab.tsx b/dashboard/components/plugins/providers-tab.tsx new file mode 100644 index 0000000..89435b1 --- /dev/null +++ b/dashboard/components/plugins/providers-tab.tsx @@ -0,0 +1,547 @@ +"use client"; + +import { useState } from "react"; +import useSWR from "swr"; +import { providerService, pluginService } from "@/lib/api-client"; +import type { ProviderDto, InstalledPlugin } from "@/lib/types"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + Plus, + RefreshCw, + Trash2, + Edit2, + KeyRound, + TestTube2, + CheckCircle2, + AlertCircle, + Loader2, + Server, +} from "lucide-react"; +import { toast } from "sonner"; + +export function ProvidersTab() { + const { + data: providersData, + isLoading: providersLoading, + mutate: mutateProviders, + } = useSWR<{ providers: ProviderDto[] }>("/api/providers", () => providerService.list()); + + const { data: pluginsData } = useSWR<{ plugins: InstalledPlugin[] }>( + "/api/plugins/installed", + () => pluginService.listInstalled() + ); + + const [modalOpen, setModalOpen] = useState(false); + const [editingProvider, setEditingProvider] = useState(null); + + // Form State + const [providerId, setProviderId] = useState(""); + const [pluginName, setPluginName] = useState(""); + const [priority, setPriority] = useState(0); + const [enabled, setEnabled] = useState(true); + const [credentials, setCredentials] = useState>({}); + const [saving, setSaving] = useState(false); + + // Test Connection State + const [testing, setTesting] = useState(false); + const [testResult, setTestResult] = useState<{ success: boolean; message: string } | null>(null); + + // Delete State + const [deleteDialogOpen, setDeleteDialogOpen] = useState(false); + const [providerToDelete, setProviderToDelete] = useState(null); + const [deleting, setDeleting] = useState(false); + + const installedPlugins = pluginsData?.plugins || []; + const selectedPlugin = installedPlugins.find((p) => p.name === pluginName); + + const resetForm = () => { + setEditingProvider(null); + setProviderId(""); + setPluginName(installedPlugins[0]?.name || ""); + setPriority(0); + setEnabled(true); + setCredentials({}); + setTestResult(null); + }; + + const handleOpenCreate = () => { + resetForm(); + if (installedPlugins.length > 0) { + setPluginName(installedPlugins[0].name); + } + setModalOpen(true); + }; + + const handleOpenEdit = async (provider: ProviderDto) => { + setEditingProvider(provider); + setProviderId(provider.id); + setPluginName(provider.plugin_name); + setPriority(provider.priority); + setEnabled(provider.enabled); + setCredentials({}); + setTestResult(null); + setModalOpen(true); + }; + + const handleTestConnection = async () => { + setTesting(true); + setTestResult(null); + try { + const result = await providerService.test({ + provider_id: editingProvider ? editingProvider.id : undefined, + plugin_name: pluginName, + credentials: Object.keys(credentials).length > 0 ? credentials : undefined, + }); + setTestResult(result); + if (result.success) { + toast.success("Health check succeeded!"); + } else { + toast.error(result.message || "Health check failed"); + } + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Connection test failed"; + setTestResult({ success: false, message }); + toast.error(message); + } finally { + setTesting(false); + } + }; + + const handleSave = async (e: React.FormEvent) => { + e.preventDefault(); + setSaving(true); + try { + if (editingProvider) { + await providerService.update(editingProvider.id, { + priority, + enabled, + credentials: Object.keys(credentials).length > 0 ? credentials : undefined, + }); + toast.success(`Provider '${editingProvider.id}' updated`); + } else { + if (!providerId.trim()) { + toast.error("Provider ID is required"); + return; + } + await providerService.create({ + id: providerId.trim(), + plugin_name: pluginName, + credentials, + priority, + enabled, + }); + toast.success(`Provider '${providerId}' created`); + } + setModalOpen(false); + resetForm(); + mutateProviders(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to save provider"; + toast.error(message); + } finally { + setSaving(false); + } + }; + + const handleDelete = async () => { + if (!providerToDelete) return; + setDeleting(true); + try { + await providerService.delete(providerToDelete); + toast.success(`Provider '${providerToDelete}' deleted`); + setDeleteDialogOpen(false); + setProviderToDelete(null); + mutateProviders(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to delete provider"; + toast.error(message); + } finally { + setDeleting(false); + } + }; + + const handleToggleEnabled = async (provider: ProviderDto, newEnabled: boolean) => { + try { + await providerService.update(provider.id, { enabled: newEnabled }); + toast.success(`Provider '${provider.id}' ${newEnabled ? "enabled" : "disabled"}`); + mutateProviders(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to update provider status"; + toast.error(message); + } + }; + + const providers = providersData?.providers || []; + + return ( +
+ {/* Header */} +
+
+

Configured Providers

+

+ Configure provider instances with envelope-encrypted credentials and priorities. +

+
+
+ + +
+
+ + {installedPlugins.length === 0 && ( + + + + + No plugins are installed yet. Please switch to the Installed Plugins{" "} + tab and install a plugin package first before creating providers. + + + + )} + + {/* Providers Grid */} + {providersLoading ? ( +
+ {[1, 2, 3].map((i) => ( + + + + + + + + ))} +
+ ) : providers.length === 0 ? ( + + + +
+

No providers configured

+

+ Create a provider instance for your installed plugins. +

+
+ + + + ) : ( +
+ {providers.map((p) => ( + + +
+
+

{p.id}

+

+ {p.plugin_name} +

+
+ + {p.enabled ? "Enabled" : "Disabled"} + +
+ +
+ Channel: {p.channel} + Priority: {p.priority} +
+ + Encrypted (RSA-OAEP) +
+
+ +
+
+ handleToggleEnabled(p, checked)} + /> + Active +
+ +
+ + +
+
+ + + ))} +
+ )} + + {/* Add/Edit Provider Modal */} + + +
+ + {editingProvider ? "Edit Provider" : "Configure Provider"} + + {editingProvider + ? `Update settings or rotate encrypted credentials for ${editingProvider.id}.` + : "Create an active provider instance backed by an installed plugin."} + + + +
+ {!editingProvider ? ( + <> +
+ + +
+ +
+ + setProviderId(e.target.value)} + required + /> +
+ + ) : ( +
+
+ Provider ID: {editingProvider.id} +
+
+ Plugin: {editingProvider.plugin_name} +
+
+ Channel: {editingProvider.channel} +
+
+ )} + +
+
+ + setPriority(parseInt(e.target.value, 10) || 0)} + /> +
+
+
+ + +
+
+
+ + {/* Dynamic Credential Fields from Plugin Manifest */} +
+
+ + + + Envelope Encrypted + +
+ + {selectedPlugin?.manifest?.requiredCredentials && + selectedPlugin.manifest.requiredCredentials.length > 0 ? ( + selectedPlugin.manifest.requiredCredentials.map((field) => ( +
+ + + setCredentials({ ...credentials, [field]: e.target.value }) + } + required={!editingProvider} + /> +
+ )) + ) : ( +

+ This plugin does not specify required credentials. +

+ )} +
+ + {/* Live Test Connection Result */} + {testResult && ( +
+ {testResult.success ? ( + + ) : ( + + )} + {testResult.message} +
+ )} +
+ + + + +
+ + +
+ +
+ + + + {/* Delete Confirmation Dialog */} + + + + Delete Provider + + Are you sure you want to delete {providerToDelete}? This will unregister the instance. + Deletion will be blocked if this provider is assigned in channel routing. + + + + Cancel + { + e.preventDefault(); + handleDelete(); + }} + disabled={deleting} + className="bg-destructive text-destructive-foreground hover:bg-destructive/90" + > + {deleting ? "Deleting..." : "Delete Provider"} + + + + +
+ ); +} diff --git a/dashboard/lib/api-client.ts b/dashboard/lib/api-client.ts index 7d6fa92..1dd95b7 100644 --- a/dashboard/lib/api-client.ts +++ b/dashboard/lib/api-client.ts @@ -19,6 +19,9 @@ import { AdminChannel, AdminChannelFormData, AdminChannelProviderMeta, + InstalledPlugin, + ProviderDto, + ChannelRoutingDto, } from './types'; export class ApiError extends Error { @@ -128,6 +131,57 @@ export const alertService = { export const pluginService = { getMetadata: (): Promise => apiClient.get('/api/plugins'), + listInstalled: (): Promise<{ plugins: InstalledPlugin[] }> => apiClient.get('/api/plugins/installed'), + install: (packageName: string, version?: string): Promise<{ message: string; plugin: InstalledPlugin }> => + apiClient.post('/api/plugins/install', { package: packageName, version }), + changeVersion: (packageName: string, version: string): Promise<{ message: string; plugin: InstalledPlugin }> => + apiClient.put('/api/plugins/version', { package: packageName, version }), + uninstall: (packageName: string): Promise<{ message: string }> => + apiClient.delete(`/api/plugins/${encodeURIComponent(packageName)}`), +}; + +export const providerService = { + list: (): Promise<{ providers: ProviderDto[] }> => apiClient.get('/api/providers'), + get: (id: string, includeDecrypted = false): Promise<{ provider: ProviderDto }> => + apiClient.get(`/api/providers/${id}`, { params: { include_decrypted: includeDecrypted } }), + create: (payload: { + id: string; + plugin_name: string; + credentials: Record; + priority?: number; + options?: Record; + enabled?: boolean; + }): Promise<{ message: string; provider: ProviderDto }> => + apiClient.post('/api/providers', payload), + update: (id: string, payload: { + credentials?: Record; + priority?: number; + options?: Record; + enabled?: boolean; + }): Promise<{ message: string; provider: ProviderDto }> => + apiClient.put(`/api/providers/${id}`, payload), + delete: (id: string): Promise<{ message: string }> => + apiClient.delete(`/api/providers/${id}`), + test: (payload: { + provider_id?: string; + plugin_name?: string; + credentials?: Record; + options?: Record; + }): Promise<{ success: boolean; message: string }> => + apiClient.post('/api/providers/test', payload), +}; + +export const channelRoutingService = { + list: (): Promise<{ routings: ChannelRoutingDto[] }> => apiClient.get('/api/channels/routing'), + get: (channel: string): Promise<{ routing: ChannelRoutingDto }> => apiClient.get(`/api/channels/routing/${channel}`), + set: (channel: string, payload: { + default_provider_id: string; + fallback_provider_ids?: string[]; + partitions?: number; + }): Promise<{ message: string; routing: ChannelRoutingDto }> => + apiClient.put(`/api/channels/routing/${channel}`, payload), + delete: (channel: string): Promise<{ message: string }> => + apiClient.delete(`/api/channels/routing/${channel}`), }; export const adminChannelService = { diff --git a/dashboard/lib/types.ts b/dashboard/lib/types.ts index d986947..35126a8 100644 --- a/dashboard/lib/types.ts +++ b/dashboard/lib/types.ts @@ -216,3 +216,54 @@ export interface AdminChannelProviderMeta { displayName: string; credentialFields: AdminChannelProviderField[]; } + +// ============================================================================ +// DYNAMIC PLUGIN & PROVIDER MANAGEMENT TYPES +// ============================================================================ + +export interface ProviderManifest { + name: string; + displayName: string; + version: string; + channel: string; + description: string; + author?: string; + homepage?: string; + requiredCredentials: string[]; + optionalConfig?: string[]; +} + +export interface InstalledPlugin { + _id?: string; + name: string; + version: string; + status: 'installed' | 'installing' | 'failed'; + error?: string; + manifest: ProviderManifest; + created_at?: string; + updated_at?: string; +} + +export interface ProviderDto { + _id?: string; + id: string; + plugin_name: string; + channel: string; + priority: number; + enabled: boolean; + options?: Record; + credentials_configured: boolean; + decrypted_credentials?: Record; + created_at?: string; + updated_at?: string; +} + +export interface ChannelRoutingDto { + _id?: string; + channel: string; + default_provider_id: string; + fallback_provider_ids: string[]; + partitions: number; + created_at?: string; + updated_at?: string; +} diff --git a/src/api/controllers/channel-routing.controller.ts b/src/api/controllers/channel-routing.controller.ts new file mode 100644 index 0000000..f625f43 --- /dev/null +++ b/src/api/controllers/channel-routing.controller.ts @@ -0,0 +1,125 @@ +/** + * Channel Routing Controller + * + * Manages channel default and fallback provider routing and Kafka partition scaling + */ + +import { Request, Response } from 'express'; +import { ChannelRoutingService } from '@src/plugins/index.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +/** + * GET /api/channels/routing + * List all channel routings + */ +export const listChannelRoutings = async (_req: Request, res: Response): Promise => { + try { + const routings = await ChannelRoutingService.listChannelRoutings(); + res.json({ routings }); + } catch (err) { + logger.error('Error listing channel routings:', err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to list channel routings' + }); + } +}; + +/** + * GET /api/channels/routing/:channel + * Get routing for a specific channel + */ +export const getChannelRouting = async (req: Request, res: Response): Promise => { + try { + const routing = await ChannelRoutingService.getChannelRouting(req.params.channel); + if (!routing) { + res.status(404).json({ + error: 'Not Found', + message: `Routing for channel '${req.params.channel}' not found` + }); + return; + } + + res.json({ routing }); + } catch (err) { + logger.error(`Error getting channel routing for '${req.params.channel}':`, err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to retrieve channel routing' + }); + } +}; + +/** + * PUT /api/channels/routing/:channel + * Set or update routing for a channel + */ +export const setChannelRouting = async (req: Request, res: Response): Promise => { + try { + const channel = req.params.channel.trim(); + const { default_provider_id, fallback_provider_ids, partitions } = req.body; + + if (!default_provider_id || typeof default_provider_id !== 'string') { + res.status(400).json({ + error: 'Bad Request', + message: 'default_provider_id is required' + }); + return; + } + + if (fallback_provider_ids !== undefined && !Array.isArray(fallback_provider_ids)) { + res.status(400).json({ + error: 'Bad Request', + message: 'fallback_provider_ids must be an array of provider IDs' + }); + return; + } + + if (partitions !== undefined && (typeof partitions !== 'number' || partitions < 1)) { + res.status(400).json({ + error: 'Bad Request', + message: 'partitions must be a positive integer' + }); + return; + } + + const routing = await ChannelRoutingService.setChannelRouting(channel, { + default_provider_id, + fallback_provider_ids, + partitions, + }); + + res.json({ + message: `Routing for channel '${channel}' updated successfully`, + routing + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error updating channel routing for '${req.params.channel}': ${message}`); + const statusCode = message.includes('Cannot decrease partitions') || message.includes('not found') ? 400 : 500; + res.status(statusCode).json({ + error: 'Failed to set channel routing', + message + }); + } +}; + +/** + * DELETE /api/channels/routing/:channel + * Delete routing for a channel + */ +export const deleteChannelRouting = async (req: Request, res: Response): Promise => { + try { + await ChannelRoutingService.deleteChannelRouting(req.params.channel); + res.json({ + message: `Routing for channel '${req.params.channel}' deleted successfully` + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error deleting channel routing for '${req.params.channel}': ${message}`); + res.status(500).json({ + error: 'Failed to delete channel routing', + message + }); + } +}; diff --git a/src/api/controllers/plugins.controller.ts b/src/api/controllers/plugins.controller.ts index 37f44be..5275d26 100644 --- a/src/api/controllers/plugins.controller.ts +++ b/src/api/controllers/plugins.controller.ts @@ -1,34 +1,150 @@ /** * Plugins Controller * - * Exposes plugin metadata for dashboard dynamic form generation + * Exposes plugin metadata, package management, and status */ import { Request, Response } from 'express'; -import { PluginRegistry } from '@src/plugins/index.js'; +import { PluginRegistry, PluginManagerService } from '@src/plugins/index.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; /** - * GET /plugins + * GET /api/plugins * Returns available channels and their provider schemas */ export const getPluginsMetadata = async (_req: Request, res: Response): Promise => { - try { - if (!PluginRegistry.isInitialized()) { - res.status(503).json({ - error: 'Plugin system not initialized', - message: 'The plugin system has not been initialized yet. Ensure providers are loaded.' - }); - return; - } - - const metadata = PluginRegistry.getPluginMetadata(); - res.json(metadata); - } catch (err) { - logger.error('Error getting plugin metadata:', err); - res.status(500).json({ - error: 'Internal server error', - message: 'Failed to retrieve plugin metadata' - }); + try { + if (!PluginRegistry.isInitialized()) { + res.status(503).json({ + error: 'Plugin system not initialized', + message: 'The plugin system has not been initialized yet. Ensure providers are loaded.' + }); + return; } + + const metadata = PluginRegistry.getPluginMetadata(); + res.json(metadata); + } catch (err) { + logger.error('Error getting plugin metadata:', err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to retrieve plugin metadata' + }); + } +}; + +/** + * GET /api/plugins/installed + * List all installed npm plugins in MongoDB + */ +export const listInstalledPlugins = async (_req: Request, res: Response): Promise => { + try { + const plugins = await PluginManagerService.listInstalledPlugins(); + res.json({ plugins }); + } catch (err) { + logger.error('Error listing installed plugins:', err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to list installed plugins' + }); + } +}; + +/** + * POST /api/plugins/install + * Install an npm plugin package + */ +export const installPlugin = async (req: Request, res: Response): Promise => { + try { + const { package: packageName, version } = req.body; + if (!packageName || typeof packageName !== 'string') { + res.status(400).json({ + error: 'Bad Request', + message: 'A valid npm package name is required' + }); + return; + } + + const plugin = await PluginManagerService.installPlugin(packageName.trim(), version); + res.status(201).json({ + message: `Plugin '${packageName}' installed successfully`, + plugin + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error installing plugin: ${message}`); + res.status(500).json({ + error: 'Failed to install plugin', + message + }); + } +}; + +/** + * PUT /api/plugins/version + * Upgrade or downgrade an installed plugin + */ +export const changePluginVersion = async (req: Request, res: Response): Promise => { + try { + const { package: packageName, version } = req.body; + if (!packageName || !version) { + res.status(400).json({ + error: 'Bad Request', + message: 'Both package name and target version are required' + }); + return; + } + + const plugin = await PluginManagerService.changePluginVersion(packageName.trim(), version.trim()); + res.json({ + message: `Plugin '${packageName}' changed to version '${version}'`, + plugin + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error changing plugin version: ${message}`); + res.status(500).json({ + error: 'Failed to change plugin version', + message + }); + } +}; + +/** + * DELETE /api/plugins/:package + * Uninstall an installed plugin + */ +export const uninstallPlugin = async (req: Request, res: Response): Promise => { + try { + let packageName = (req.query.package as string) || (req.body?.package as string); + + if (!packageName) { + if (req.params.scope && req.params.package) { + packageName = `${req.params.scope}/${req.params.package}`; + } else if (req.params.package) { + packageName = decodeURIComponent(req.params.package); + } + } + + if (!packageName || typeof packageName !== 'string') { + res.status(400).json({ + error: 'Bad Request', + message: 'A valid npm package name is required' + }); + return; + } + + await PluginManagerService.uninstallPlugin(packageName.trim()); + res.json({ + message: `Plugin '${packageName.trim()}' uninstalled successfully` + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error uninstalling plugin: ${message}`); + const statusCode = message.includes('depend on it') ? 400 : 500; + res.status(statusCode).json({ + error: 'Failed to uninstall plugin', + message + }); + } }; diff --git a/src/api/controllers/providers.controller.ts b/src/api/controllers/providers.controller.ts new file mode 100644 index 0000000..5164366 --- /dev/null +++ b/src/api/controllers/providers.controller.ts @@ -0,0 +1,190 @@ +/** + * Providers Controller + * + * Manages provider configurations, credential testing, and lifecycle + */ + +import { Request, Response } from 'express'; +import { ProviderManagerService } from '@src/plugins/index.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +/** + * GET /api/providers + * List all configured providers + */ +export const listProviders = async (_req: Request, res: Response): Promise => { + try { + const providers = await ProviderManagerService.listProviders(); + res.json({ providers }); + } catch (err) { + logger.error('Error listing providers:', err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to list providers' + }); + } +}; + +/** + * GET /api/providers/:id + * Get single provider by ID + */ +export const getProvider = async (req: Request, res: Response): Promise => { + try { + const includeDecrypted = req.query.include_decrypted === 'true'; + const provider = await ProviderManagerService.getProvider(req.params.id, includeDecrypted); + + if (!provider) { + res.status(404).json({ + error: 'Not Found', + message: `Provider '${req.params.id}' not found` + }); + return; + } + + res.json({ provider }); + } catch (err) { + logger.error(`Error getting provider '${req.params.id}':`, err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to retrieve provider' + }); + } +}; + +/** + * POST /api/providers + * Create a new provider instance + */ +export const createProvider = async (req: Request, res: Response): Promise => { + try { + const { id, plugin_name, credentials, priority, options, enabled } = req.body; + + if (!id || typeof id !== 'string') { + res.status(400).json({ + error: 'Bad Request', + message: 'Provider id is required and must be a string' + }); + return; + } + + if (!plugin_name || typeof plugin_name !== 'string') { + res.status(400).json({ + error: 'Bad Request', + message: 'plugin_name is required' + }); + return; + } + + if (!credentials || typeof credentials !== 'object') { + res.status(400).json({ + error: 'Bad Request', + message: 'credentials object is required' + }); + return; + } + + const provider = await ProviderManagerService.createProvider({ + id: id.trim(), + plugin_name: plugin_name.trim(), + credentials, + priority: typeof priority === 'number' ? priority : 0, + options: options || {}, + enabled: enabled !== false, + }); + + res.status(201).json({ + message: `Provider '${id}' created successfully`, + provider + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error creating provider: ${message}`); + const statusCode = message.includes('already exists') ? 409 : message.includes('not installed') ? 404 : 500; + res.status(statusCode).json({ + error: 'Failed to create provider', + message + }); + } +}; + +/** + * PUT /api/providers/:id + * Update an existing provider instance + */ +export const updateProvider = async (req: Request, res: Response): Promise => { + try { + const { credentials, priority, options, enabled } = req.body; + + const provider = await ProviderManagerService.updateProvider(req.params.id, { + credentials, + priority: typeof priority === 'number' ? priority : undefined, + options, + enabled: typeof enabled === 'boolean' ? enabled : undefined, + }); + + res.json({ + message: `Provider '${req.params.id}' updated successfully`, + provider + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error updating provider '${req.params.id}': ${message}`); + const statusCode = message.includes('not found') ? 404 : 500; + res.status(statusCode).json({ + error: 'Failed to update provider', + message + }); + } +}; + +/** + * DELETE /api/providers/:id + * Delete a provider instance + */ +export const deleteProvider = async (req: Request, res: Response): Promise => { + try { + await ProviderManagerService.deleteProvider(req.params.id); + res.json({ + message: `Provider '${req.params.id}' deleted successfully` + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error deleting provider '${req.params.id}': ${message}`); + const statusCode = message.includes('configured in channel') ? 400 : message.includes('not found') ? 404 : 500; + res.status(statusCode).json({ + error: 'Failed to delete provider', + message + }); + } +}; + +/** + * POST /api/providers/test + * Test provider connection / credentials + */ +export const testProvider = async (req: Request, res: Response): Promise => { + try { + const { provider_id, plugin_name, credentials, options } = req.body; + + const result = await ProviderManagerService.testProviderConnection({ + provider_id, + plugin_name, + credentials, + options, + }); + + if (result.success) { + res.json(result); + } else { + res.status(400).json(result); + } + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error testing provider: ${message}`); + res.status(500).json({ + success: false, + message + }); + } +}; diff --git a/src/api/routes/channel-routing.routes.ts b/src/api/routes/channel-routing.routes.ts new file mode 100644 index 0000000..2592505 --- /dev/null +++ b/src/api/routes/channel-routing.routes.ts @@ -0,0 +1,22 @@ +/** + * Channel Routing Routes + * + * Endpoints for configuring default and cascading fallback provider chains + */ + +import { Router } from 'express'; +import { + listChannelRoutings, + getChannelRouting, + setChannelRouting, + deleteChannelRouting, +} from '../controllers/channel-routing.controller.js'; + +const router = Router(); + +router.get('/', listChannelRoutings); +router.get('/:channel', getChannelRouting); +router.put('/:channel', setChannelRouting); +router.delete('/:channel', deleteChannelRouting); + +export default router; diff --git a/src/api/routes/plugins.routes.ts b/src/api/routes/plugins.routes.ts index e0adaa8..afdf88c 100644 --- a/src/api/routes/plugins.routes.ts +++ b/src/api/routes/plugins.routes.ts @@ -1,18 +1,26 @@ /** * Plugins Routes * - * Exposes plugin metadata for dashboard + * Exposes plugin metadata, installation, update, and uninstallation endpoints */ import { Router } from 'express'; -import { getPluginsMetadata } from '../controllers/plugins.controller.js'; +import { + getPluginsMetadata, + listInstalledPlugins, + installPlugin, + changePluginVersion, + uninstallPlugin, +} from '../controllers/plugins.controller.js'; const router = Router(); -/** - * GET /plugins - * Returns available channels and their provider schemas - */ router.get('/', getPluginsMetadata); +router.get('/installed', listInstalledPlugins); +router.post('/install', installPlugin); +router.put('/version', changePluginVersion); +router.delete('/:scope/:package', uninstallPlugin); +router.delete('/:package', uninstallPlugin); +router.delete('/', uninstallPlugin); export default router; diff --git a/src/api/routes/providers.routes.ts b/src/api/routes/providers.routes.ts new file mode 100644 index 0000000..6f6045f --- /dev/null +++ b/src/api/routes/providers.routes.ts @@ -0,0 +1,26 @@ +/** + * Providers Routes + * + * Endpoints for managing provider instances and testing connections + */ + +import { Router } from 'express'; +import { + listProviders, + getProvider, + createProvider, + updateProvider, + deleteProvider, + testProvider, +} from '../controllers/providers.controller.js'; + +const router = Router(); + +router.get('/', listProviders); +router.post('/test', testProvider); +router.get('/:id', getProvider); +router.post('/', createProvider); +router.put('/:id', updateProvider); +router.delete('/:id', deleteProvider); + +export default router; diff --git a/src/api/server.ts b/src/api/server.ts index 0b4d26d..c03e256 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -5,6 +5,8 @@ import { connectMongoDB } from '@src/config/db.config.js'; import { exit } from 'process'; import notification_router from './routes/notification.routes.js'; import plugins_router from './routes/plugins.routes.js'; +import providers_router from './routes/providers.routes.js'; +import channel_routing_router from './routes/channel-routing.routes.js'; import admin_channels_router from './routes/admin-channels.routes.js'; import notification_templates_router from '@src/api/routes/notification_templates.routes.js'; import notifications_management_router from './routes/notifications-management.routes.js'; @@ -14,10 +16,16 @@ import { auth_middleware } from './middlewares/auth_middleware.js'; import http from 'http'; import helmet from 'helmet'; import cors from 'cors'; -import { createTopics } from '@src/config/kafka.config.js'; +import { createTopics, buildKafkaTopicsFromDatabase } from '@src/config/kafka.config.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; -import { buildKafkaTopics } from '@src/config/kafka.config.js'; -import { loadProvidersFromEnv } from '@src/plugins/index.js'; +import { + PluginSyncService, + PluginManagerService, + ProviderManagerService, + ChannelRoutingService, + YamlMigrator, + loadProvidersFromDatabase, +} from '@src/plugins/index.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; //Import the admin channel provider files here for them to self-register @@ -55,6 +63,8 @@ app.use('/api/notifications', auth_middleware, notifications_management_router); app.use('/api/alerts', auth_middleware, alerts_router); app.use('/api/dashboard', auth_middleware, dashboard_router); app.use('/api/plugins', auth_middleware, plugins_router); +app.use('/api/providers', auth_middleware, providers_router); +app.use('/api/channels/routing', auth_middleware, channel_routing_router); app.use('/api/admin-channels', auth_middleware, admin_channels_router); app.use('/api/templates', auth_middleware, notification_templates_router); @@ -63,14 +73,21 @@ const start_server = async () => { const db = await connectMongoDB(); logger.success("Successfully connected to MongoDB"); - // Load plugins from simplens.config.yaml - // Initialize: false is important here because the API service doesn't need to connect to providers (e.g. SMTP), - // it only needs the metadata/schemas to serve the dashboard. - logger.info('Loading plugins from configuration (metadata only)...'); - await loadProvidersFromEnv({ initialize: false }); + // 1. Initialize Redis subscriber & sync handlers for plugins + await PluginSyncService.startSubscriber(); + PluginManagerService.registerSyncHandlers(); + ProviderManagerService.registerSyncHandlers(); + ChannelRoutingService.registerSyncHandlers(); - // Create Kafka topics dynamically from config - const topics = buildKafkaTopics(); + // 2. Check if MongoDB has providers; if empty and YAML exists, migrate it + await YamlMigrator.migrateYamlIfNeeded(); + + // 3. Load provider plugins from MongoDB (metadata only for API server) + logger.info('Loading plugins from MongoDB (metadata only)...'); + await loadProvidersFromDatabase({ initialize: false }); + + // 4. Create Kafka topics dynamically from MongoDB channel routing + const topics = await buildKafkaTopicsFromDatabase(); await createTopics(topics); const server = http.createServer(app); diff --git a/src/config/kafka.config.ts b/src/config/kafka.config.ts index 2fb5c02..9b4c313 100644 --- a/src/config/kafka.config.ts +++ b/src/config/kafka.config.ts @@ -52,6 +52,45 @@ export const buildKafkaTopics = (): { topic: string; numPartitions: number; repl return topics; }; +/** + * Build Kafka topics dynamically from MongoDB channel routing records + */ +export const buildKafkaTopicsFromDatabase = async (): Promise => { + const topics: ITopicConfig[] = [ + { + topic: CORE_TOPICS.delayed_notification, + numPartitions: env.DELAYED_PARTITION, + replicationFactor: 1 + }, + { + topic: CORE_TOPICS.notification_status, + numPartitions: env.NOTIFICATION_STATUS_PARTITION, + replicationFactor: 1 + }, + ]; + + try { + const ChannelRouting = (await import('@src/database/models/channel-routing.models.js')).default; + const routings = await ChannelRouting.find().lean(); + if (routings && routings.length > 0) { + for (const r of routings) { + topics.push({ + topic: getTopicForChannel(r.channel), + numPartitions: r.partitions || 6, + replicationFactor: 1, + }); + logger.info(`Configured topic from DB: ${getTopicForChannel(r.channel)} (${r.partitions || 6} partitions)`); + } + return topics; + } + } catch (err) { + logger.warn('Could not query ChannelRouting for Kafka topics, falling back to static config:', { + error: err instanceof Error ? err.message : String(err) + }); + } + + return buildKafkaTopics(); +}; export const createTopics = async (topics: ITopicConfig[]) => { const admin = kafka.admin(); @@ -77,10 +116,19 @@ export const createTopics = async (topics: ITopicConfig[]) => { const currentPartitions = topicMetadata.partitions.length; const desiredPartitions = configuredTopic.numPartitions; - if (currentPartitions !== desiredPartitions) { + if (desiredPartitions > currentPartitions) { + logger.info(`Expanding topic "${topicMetadata.name}" from ${currentPartitions} to ${desiredPartitions} partitions...`); + await admin.createPartitions({ + topicPartitions: [{ + topic: topicMetadata.name, + count: desiredPartitions + }] + }); + logger.success(`Successfully expanded topic "${topicMetadata.name}" to ${desiredPartitions} partitions`); + } else if (desiredPartitions < currentPartitions) { logger.info( - `Topic "${topicMetadata.name}" has ${currentPartitions} partitions but ${desiredPartitions} is configured. ` + - `Partition count cannot be changed after creation. Delete and recreate the topic to apply changes.` + `Topic "${topicMetadata.name}" has ${currentPartitions} partitions (configured: ${desiredPartitions}). ` + + `Kafka partition count cannot be decreased.` ); } } @@ -116,3 +164,48 @@ export const createTopics = async (topics: ITopicConfig[]) => { } }; +/** + * Dynamically expand partition count for a channel topic + */ +export const expandTopicPartitions = async ( + channel: string, + desiredPartitions: number +): Promise<{ previous: number; current: number }> => { + const admin = kafka.admin(); + const topic = getTopicForChannel(channel); + + try { + await admin.connect(); + const metadata = await admin.fetchTopicMetadata({ topics: [topic] }); + const topicMetadata = metadata.topics.find(t => t.name === topic); + + if (!topicMetadata) { + throw new Error(`Topic '${topic}' for channel '${channel}' not found in Kafka`); + } + + const currentPartitions = topicMetadata.partitions.length; + + if (desiredPartitions < currentPartitions) { + throw new Error( + `Kafka partition count cannot be decreased. Current count is ${currentPartitions}, requested ${desiredPartitions}.` + ); + } + + if (desiredPartitions > currentPartitions) { + logger.info(`Expanding Kafka topic '${topic}' from ${currentPartitions} to ${desiredPartitions} partitions...`); + await admin.createPartitions({ + topicPartitions: [{ + topic, + count: desiredPartitions + }] + }); + logger.success(`Expanded Kafka topic '${topic}' to ${desiredPartitions} partitions`); + return { previous: currentPartitions, current: desiredPartitions }; + } + + return { previous: currentPartitions, current: currentPartitions }; + } finally { + await admin.disconnect(); + } +}; + diff --git a/src/database/models/admin-channel.models.ts b/src/database/models/admin-channel.models.ts index aa66a48..6bc541c 100644 --- a/src/database/models/admin-channel.models.ts +++ b/src/database/models/admin-channel.models.ts @@ -59,9 +59,11 @@ const admin_channel_schema = new mongoose.Schema( } ); -const admin_channel_model = mongoose.model( - 'AdminNotificationChannel', - admin_channel_schema -); +const admin_channel_model = + mongoose.models.AdminNotificationChannel || + mongoose.model( + 'AdminNotificationChannel', + admin_channel_schema + ); export default admin_channel_model; diff --git a/src/database/models/channel-routing.models.ts b/src/database/models/channel-routing.models.ts new file mode 100644 index 0000000..1cb5c76 --- /dev/null +++ b/src/database/models/channel-routing.models.ts @@ -0,0 +1,45 @@ +/** + * Channel Routing Model + * Stores default and cascading fallback provider chains, plus Kafka partition targets + */ + +import { type channel_routing_document } from '@src/types/types.js'; +import mongoose from 'mongoose'; + +const channel_routing_schema = new mongoose.Schema( + { + channel: { + type: String, + required: true, + unique: true, + index: true, + }, + default_provider_id: { + type: String, + required: true, + }, + fallback_provider_ids: { + type: [String], + default: [], + }, + partitions: { + type: Number, + default: 6, + min: 1, + }, + }, + { + timestamps: { + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + } +); + +const channel_routing_model = + mongoose.models.ChannelRouting || mongoose.model( + 'ChannelRouting', + channel_routing_schema + ); + +export default channel_routing_model; diff --git a/src/database/models/plugin.models.ts b/src/database/models/plugin.models.ts new file mode 100644 index 0000000..57965dd --- /dev/null +++ b/src/database/models/plugin.models.ts @@ -0,0 +1,61 @@ +/** + * Plugin Model + * Stores installed notification plugins with metadata and package version + */ + +import { type plugin_document, type provider_manifest } from '@src/types/types.js'; +import mongoose from 'mongoose'; + +const manifestSubSchema = new mongoose.Schema( + { + name: { type: String, required: true }, + version: { type: String, required: true }, + channel: { type: String, required: true }, + displayName: { type: String, required: true }, + description: { type: String, required: true }, + author: { type: String }, + homepage: { type: String }, + requiredCredentials: [{ type: String, required: true }], + optionalConfig: [{ type: String }], + }, + { _id: false } +); + +const plugin_schema = new mongoose.Schema( + { + name: { + type: String, + required: true, + unique: true, + index: true, + }, + version: { + type: String, + required: true, + }, + status: { + type: String, + enum: ['installed', 'installing', 'failed'], + default: 'installed', + index: true, + }, + error: { + type: String, + }, + manifest: { + type: manifestSubSchema, + required: true, + }, + }, + { + timestamps: { + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + } +); + +const plugin_model = + mongoose.models.Plugin || mongoose.model('Plugin', plugin_schema); + +export default plugin_model; diff --git a/src/database/models/provider.models.ts b/src/database/models/provider.models.ts new file mode 100644 index 0000000..2659d3b --- /dev/null +++ b/src/database/models/provider.models.ts @@ -0,0 +1,74 @@ +/** + * Provider Model + * Stores configured provider instances with encrypted credentials + */ + +import { + type provider_document, + type encrypted_credentials, +} from '@src/types/types.js'; +import mongoose from 'mongoose'; + +// Subdocument schema for key-pair envelope encrypted credentials +const encryptedCredentialsSubSchema = new mongoose.Schema( + { + encrypted_data: { type: String, required: true }, + iv: { type: String, required: true }, + auth_tag: { type: String, required: true }, + encrypted_dek: { type: String, required: true }, + }, + { _id: false } +); + +const provider_schema = new mongoose.Schema( + { + id: { + type: String, + required: true, + unique: true, + index: true, + }, + plugin_name: { + type: String, + required: true, + index: true, + }, + channel: { + type: String, + required: true, + index: true, + }, + priority: { + type: Number, + default: 0, + index: true, + }, + enabled: { + type: Boolean, + default: true, + index: true, + }, + credentials: { + type: encryptedCredentialsSubSchema, + required: true, + }, + options: { + type: mongoose.Schema.Types.Mixed, + default: {}, + }, + }, + { + timestamps: { + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + } +); + +// Compound index for priority-based provider routing per channel +provider_schema.index({ channel: 1, priority: -1 }); + +const provider_model = + mongoose.models.Provider || mongoose.model('Provider', provider_schema); + +export default provider_model; diff --git a/src/database/models/system-config.models.ts b/src/database/models/system-config.models.ts index 13afa69..d4a7093 100644 --- a/src/database/models/system-config.models.ts +++ b/src/database/models/system-config.models.ts @@ -27,6 +27,8 @@ const system_config_schema = new mongoose.Schema( } ); -const system_config_model = mongoose.model('SystemConfig', system_config_schema); +const system_config_model = + mongoose.models.SystemConfig || + mongoose.model('SystemConfig', system_config_schema); export default system_config_model; diff --git a/src/plugins/bootstrap/yaml-migrator.ts b/src/plugins/bootstrap/yaml-migrator.ts new file mode 100644 index 0000000..741121a --- /dev/null +++ b/src/plugins/bootstrap/yaml-migrator.ts @@ -0,0 +1,251 @@ +/** + * YAML Configuration Migrator + * + * Migrates static simplens.config.yaml into MongoDB on first startup if MongoDB is empty. + * Resolves env placeholders in credentials, encrypts them using KeyPairManager, + * ensures npm packages are installed, extracts manifests with package.json versions, + * converts single fallbacks to array chains, and broadcasts SYSTEM_RELOAD. + */ + +import { existsSync, readFileSync } from 'fs'; +import { join } from 'path'; +import { parse as parseYaml } from 'yaml'; +import Plugin from '@src/database/models/plugin.models.js'; +import Provider from '@src/database/models/provider.models.js'; +import ChannelRouting from '@src/database/models/channel-routing.models.js'; +import { encryptCredentials } from '@src/plugins/crypto/keypair-manager.js'; +import { + installNpmPackage, + extractPackageManifest, + PLUGINS_NODE_MODULES, +} from '@src/plugins/loader/plugin-fs.js'; +import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +interface YamlProviderEntry { + package: string; + version?: string; + id: string; + credentials?: Record; + optionalConfig?: Record; + options?: { + priority?: number; + [key: string]: unknown; + }; +} + +interface YamlConfig { + providers?: YamlProviderEntry[]; + channels?: Record< + string, + { + default: string; + fallback?: string | string[]; + partitions?: number; + } + >; +} + +/** + * Resolve environment variables in credential templates + */ +function resolveCredentials(credentials?: Record): Record { + const resolved: Record = {}; + for (const [k, v] of Object.entries(credentials || {})) { + if (typeof v === 'string' && v.startsWith('${') && v.endsWith('}')) { + const envName = v.slice(2, -1); + resolved[k] = process.env[envName] || ''; + } else { + resolved[k] = v; + } + } + return resolved; +} + +/** + * Resolve optional config placeholders and merge into options + */ +function resolveOptionalConfig(optionalConfig?: Record): Record { + const resolved: Record = {}; + for (const [k, v] of Object.entries(optionalConfig || {})) { + if (typeof v === 'string' && v.startsWith('${') && v.endsWith('}')) { + const envName = v.slice(2, -1); + if (process.env[envName]) { + resolved[k] = process.env[envName]!; + } + } else if (v !== undefined) { + resolved[k] = v; + } + } + return resolved; +} + +/** + * Find local config file if it exists + */ +function findConfigPath(): string | null { + const customPath = process.env.SIMPLENS_CONFIG_PATH; + if (customPath && existsSync(customPath)) { + return customPath; + } + + const candidates = [ + './simplens.config.yaml', + './simplens.config.yml', + './simplens.config.json', + join(process.cwd(), 'simplens.config.yaml'), + join(process.cwd(), 'simplens.config.yml'), + ]; + + for (const p of candidates) { + if (existsSync(p)) return p; + } + return null; +} + +export class YamlMigrator { + /** + * Check if MongoDB has providers; if empty and config file exists, migrate to MongoDB. + * Returns true if migration was performed, false otherwise. + */ + public static async migrateYamlIfNeeded(): Promise { + const providerCount = await Provider.countDocuments(); + if (providerCount > 0) { + logger.info( + `Found ${providerCount} provider(s) in MongoDB. Dynamic database configuration active, skipping YAML migration.` + ); + return false; + } + + const configPath = findConfigPath(); + if (!configPath) { + logger.info('No simplens.config.yaml found and MongoDB is empty. Standby mode ready.'); + return false; + } + + logger.info(`MongoDB provider collection is empty. Migrating configuration from ${configPath}...`); + + let rawConfig: YamlConfig; + try { + const fileContent = readFileSync(configPath, 'utf-8'); + if (configPath.endsWith('.json')) { + rawConfig = JSON.parse(fileContent); + } else { + rawConfig = parseYaml(fileContent) as YamlConfig; + } + } catch (err) { + logger.error(`Failed to parse configuration file at ${configPath}:`, err); + return false; + } + + if (!rawConfig.providers || rawConfig.providers.length === 0) { + logger.warn('Configuration file contains no providers. Nothing to migrate.'); + return false; + } + + // 1. Process and install plugins + const processedPlugins = new Set(); + + for (const entry of rawConfig.providers) { + const packageName = entry.package; + if (!processedPlugins.has(packageName)) { + const targetVersion = entry.version || 'latest'; + const installedPath = join(PLUGINS_NODE_MODULES, packageName); + + // If not already in node_modules, install it + if (!existsSync(installedPath)) { + logger.info(`Migrating plugin: Installing '${packageName}@${targetVersion}'...`); + try { + installNpmPackage(packageName, targetVersion); + } catch (installErr) { + logger.error(`Could not install plugin package '${packageName}':`, installErr); + } + } + + try { + const { manifest, version: authoritativeVersion } = await extractPackageManifest(packageName); + await Plugin.findOneAndUpdate( + { name: packageName }, + { + name: packageName, + version: authoritativeVersion, + status: 'installed', + manifest, + error: undefined, + }, + { upsert: true, new: true } + ); + processedPlugins.add(packageName); + logger.success(`Migrated plugin '${packageName}' (version: ${authoritativeVersion})`); + } catch (manifestErr) { + logger.error(`Failed to extract manifest for '${packageName}':`, manifestErr); + } + } + } + + // 2. Process and encrypt provider configurations + for (const entry of rawConfig.providers) { + const resolvedCreds = resolveCredentials(entry.credentials); + const resolvedOpts = resolveOptionalConfig(entry.optionalConfig); + const mergedOptions = { + ...(entry.options || {}), + ...resolvedOpts, + }; + + const encryptedCreds = await encryptCredentials(resolvedCreds); + + // Find channel from installed plugin manifest + const pluginDoc = await Plugin.findOne({ name: entry.package }); + const channel = pluginDoc?.manifest?.channel || 'generic'; + + await Provider.findOneAndUpdate( + { id: entry.id }, + { + id: entry.id, + plugin_name: entry.package, + channel, + priority: entry.options?.priority ?? 0, + enabled: true, + credentials: encryptedCreds, + options: mergedOptions, + }, + { upsert: true, new: true } + ); + + logger.success(`Migrated provider '${entry.id}' (${channel})`); + } + + // 3. Process channel routing + if (rawConfig.channels) { + for (const [channel, route] of Object.entries(rawConfig.channels)) { + let fallbackArray: string[] = []; + if (Array.isArray(route.fallback)) { + fallbackArray = route.fallback; + } else if (typeof route.fallback === 'string' && route.fallback.trim().length > 0) { + fallbackArray = [route.fallback.trim()]; + } + + await ChannelRouting.findOneAndUpdate( + { channel }, + { + channel, + default_provider_id: route.default, + fallback_provider_ids: fallbackArray, + partitions: route.partitions || 6, + }, + { upsert: true, new: true } + ); + + logger.success( + `Migrated channel routing '${channel}': default=${route.default}, fallbacks=[${fallbackArray.join(', ')}]` + ); + } + } + + logger.success('YAML configuration migration completed successfully.'); + + // Broadcast SYSTEM_RELOAD event to notify other instances + await PluginSyncService.publish('SYSTEM_RELOAD', {}); + return true; + } +} diff --git a/src/plugins/crypto/keypair-manager.ts b/src/plugins/crypto/keypair-manager.ts new file mode 100644 index 0000000..ffe62eb --- /dev/null +++ b/src/plugins/crypto/keypair-manager.ts @@ -0,0 +1,173 @@ +/** + * Key-Pair Envelope Encryption Manager + * + * Secures provider credentials using RSA-OAEP (2048-bit) + AES-256-GCM envelope encryption. + * Automatically generates, caches, and persists RSA key pair in MongoDB or loads from environment. + */ + +import crypto from 'crypto'; +import system_config_model from '@src/database/models/system-config.models.js'; +import type { encrypted_credentials } from '@src/types/types.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +const PUBLIC_KEY_CONFIG = 'plugin_credentials_public_key'; +const PRIVATE_KEY_CONFIG = 'plugin_credentials_private_key'; + +let cachedPublicKey: string | null = null; +let cachedPrivateKey: string | null = null; + +/** + * Generate a new 2048-bit RSA key pair in PEM format + */ +export function generateRSAKeyPair(): { publicKey: string; privateKey: string } { + return crypto.generateKeyPairSync('rsa', { + modulusLength: 2048, + publicKeyEncoding: { + type: 'spki', + format: 'pem', + }, + privateKeyEncoding: { + type: 'pkcs8', + format: 'pem', + }, + }); +} + +/** + * Get or create RSA Public and Private Key Pair + * Priority: Environment Variables > MongoDB SystemConfig > Generate & Persist + */ +export async function getOrCreateKeyPair(): Promise<{ publicKey: string; privateKey: string }> { + if (cachedPublicKey && cachedPrivateKey) { + return { publicKey: cachedPublicKey, privateKey: cachedPrivateKey }; + } + + // 1. Check environment variables + const envPub = process.env.SIMPLENS_PLUGIN_PUBLIC_KEY; + const envPriv = process.env.SIMPLENS_PLUGIN_PRIVATE_KEY; + if (envPub && envPriv) { + cachedPublicKey = envPub.replace(/\\n/g, '\n'); + cachedPrivateKey = envPriv.replace(/\\n/g, '\n'); + return { publicKey: cachedPublicKey, privateKey: cachedPrivateKey }; + } + + // 2. Check MongoDB for existing keys + const [pubDoc, privDoc] = await Promise.all([ + system_config_model.findOne({ key: PUBLIC_KEY_CONFIG }), + system_config_model.findOne({ key: PRIVATE_KEY_CONFIG }), + ]); + + if (pubDoc?.value && privDoc?.value) { + cachedPublicKey = pubDoc.value as string; + cachedPrivateKey = privDoc.value as string; + return { publicKey: cachedPublicKey, privateKey: cachedPrivateKey }; + } + + // 3. Generate new key pair and store in MongoDB + logger.info('Generating new RSA 2048-bit key pair for provider credential encryption...'); + const keyPair = generateRSAKeyPair(); + + await Promise.all([ + system_config_model.findOneAndUpdate( + { key: PUBLIC_KEY_CONFIG }, + { value: keyPair.publicKey }, + { upsert: true, new: true } + ), + system_config_model.findOneAndUpdate( + { key: PRIVATE_KEY_CONFIG }, + { value: keyPair.privateKey }, + { upsert: true, new: true } + ), + ]); + + cachedPublicKey = keyPair.publicKey; + cachedPrivateKey = keyPair.privateKey; + logger.success('RSA key pair generated and persisted to MongoDB system_configs'); + + return { publicKey: cachedPublicKey, privateKey: cachedPrivateKey }; +} + +/** + * Encrypt credentials using hybrid envelope encryption (AES-256-GCM + RSA-OAEP) + * @param plaintext - Credentials object or JSON string to encrypt + */ +export async function encryptCredentials( + plaintext: Record | string +): Promise { + const { publicKey } = await getOrCreateKeyPair(); + const dataString = typeof plaintext === 'string' ? plaintext : JSON.stringify(plaintext); + + // 1. Generate random 256-bit AES Data Encryption Key (DEK) + const dek = crypto.randomBytes(32); + const iv = crypto.randomBytes(16); + + // 2. Encrypt plaintext credentials with AES-256-GCM + const cipher = crypto.createCipheriv('aes-256-gcm', dek, iv, { + authTagLength: 16, + }); + + let encryptedData = cipher.update(dataString, 'utf8', 'hex'); + encryptedData += cipher.final('hex'); + const authTag = cipher.getAuthTag().toString('hex'); + + // 3. Encrypt AES DEK with RSA Public Key (RSA-OAEP with SHA-256) + const encryptedDek = crypto.publicEncrypt( + { + key: publicKey, + padding: crypto.constants.RSA_PKCS1_OAEP_PADDING, + oaepHash: 'sha256', + }, + dek + ).toString('hex'); + + return { + encrypted_data: encryptedData, + iv: iv.toString('hex'), + auth_tag: authTag, + encrypted_dek: encryptedDek, + }; +} + +/** + * Decrypt credentials using hybrid envelope encryption (AES-256-GCM + RSA-OAEP) + * @param credentials - Encrypted credentials subdocument + * @returns Decrypted Record + */ +export async function decryptCredentials( + credentials: encrypted_credentials +): Promise> { + const { privateKey } = await getOrCreateKeyPair(); + + // 1. Decrypt AES DEK using RSA Private Key + const dek = crypto.privateDecrypt( + { + key: privateKey, + padding: crypto.constants.RSA_PKCS1_OAEP_PADDING, + oaepHash: 'sha256', + }, + Buffer.from(credentials.encrypted_dek, 'hex') + ); + + // 2. Decrypt ciphertext using AES-256-GCM + const decipher = crypto.createDecipheriv( + 'aes-256-gcm', + dek, + Buffer.from(credentials.iv, 'hex'), + { authTagLength: 16 } + ); + + decipher.setAuthTag(Buffer.from(credentials.auth_tag, 'hex')); + + let decrypted = decipher.update(credentials.encrypted_data, 'hex', 'utf8'); + decrypted += decipher.final('utf8'); + + return JSON.parse(decrypted) as Record; +} + +/** + * Clear cached key pair (used in tests) + */ +export function clearKeyPairCache(): void { + cachedPublicKey = null; + cachedPrivateKey = null; +} diff --git a/src/plugins/index.ts b/src/plugins/index.ts index da6e5e6..4a2f64d 100644 --- a/src/plugins/index.ts +++ b/src/plugins/index.ts @@ -16,7 +16,14 @@ export { } from './loader/registry.js'; // Loader -export { loadProviders, loadProvidersFromEnv, registerProvider, getConfiguredChannels } from './loader/loader.js'; +export { + loadProviders, + loadProvidersFromEnv, + loadProvidersFromDatabase, + getConfiguredChannels, + getConfiguredChannelsFromDatabase, + registerProvider, +} from './loader/loader.js'; // Router export { @@ -27,3 +34,17 @@ export { resolveFallbackProviderId, getRateLimitConfig } from './loader/router.js'; + +// Dynamic Management Services +export { PluginManagerService } from './services/plugin-manager.service.js'; +export { ProviderManagerService, type ProviderResponseDto } from './services/provider-manager.service.js'; +export { ChannelRoutingService } from './services/channel-routing.service.js'; +export { YamlMigrator } from './bootstrap/yaml-migrator.js'; +export { PluginSyncService } from './sync/plugin-sync.service.js'; +export { encryptCredentials, decryptCredentials } from './crypto/keypair-manager.js'; +export { + installNpmPackage, + uninstallNpmPackage, + extractPackageManifest, + importAndInstantiateProvider, +} from './loader/plugin-fs.js'; diff --git a/src/plugins/loader/loader.ts b/src/plugins/loader/loader.ts index 3d9d67d..0451e93 100644 --- a/src/plugins/loader/loader.ts +++ b/src/plugins/loader/loader.ts @@ -13,6 +13,12 @@ import { parse as parseYaml } from 'yaml'; import { PluginRegistry, type ChannelConfig } from './registry.js'; import type { SimpleNSProvider, ProviderConfig } from '../interfaces/provider.types.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; +import Plugin from '@src/database/models/plugin.models.js'; +import Provider from '@src/database/models/provider.models.js'; +import ChannelRouting from '@src/database/models/channel-routing.models.js'; +import { ProviderManagerService } from '@src/plugins/services/provider-manager.service.js'; +import { installNpmPackage } from './plugin-fs.js'; +import type { provider_document } from '@src/types/types.js'; // Plugins directory for user-installed plugins const PLUGINS_DIR = join(process.cwd(), '.plugins'); @@ -442,3 +448,82 @@ export async function registerProvider( const priority = (options?.priority as number) || 0; PluginRegistry.register(provider, id, priority); } + +/** + * Load all providers and channel routings directly from MongoDB + */ +export async function loadProvidersFromDatabase(options: { initialize?: boolean } = {}): Promise { + const shouldInitialize = options.initialize !== false; + logger.info('Loading provider plugins from MongoDB...'); + + // 1. Ensure all installed plugins exist locally + try { + const installedPlugins = await Plugin.find({ status: 'installed' }).lean(); + for (const plugin of installedPlugins) { + const pluginPath = join(PLUGINS_NODE_MODULES, plugin.name); + if (!existsSync(pluginPath)) { + logger.info(`Syncing missing plugin package locally: ${plugin.name}@${plugin.version}`); + try { + installNpmPackage(plugin.name, plugin.version); + } catch (err) { + logger.error(`Failed to install missing package ${plugin.name}:`, err); + } + } + } + } catch (dbErr) { + logger.error('Error fetching installed plugins from database:', dbErr); + } + + // 2. Load all enabled providers + try { + const providers = await Provider.find({ enabled: true }).lean(); + logger.info(`Found ${providers.length} enabled provider(s) in MongoDB.`); + + for (const providerDoc of providers) { + try { + await ProviderManagerService.loadAndRegisterProvider( + providerDoc as unknown as provider_document, + shouldInitialize + ); + } catch (err) { + logger.error(`Failed to load provider '${providerDoc.id}':`, err); + } + } + } catch (dbErr) { + logger.error('Error fetching providers from database:', dbErr); + } + + // 3. Load channel routing + try { + const routings = await ChannelRouting.find().lean(); + for (const route of routings) { + PluginRegistry.setChannelConfig(route.channel, { + default: route.default_provider_id, + fallback: route.fallback_provider_ids || [], + }); + } + } catch (dbErr) { + logger.error('Error fetching channel routings from database:', dbErr); + } + + PluginRegistry.setInitialized(true); + logger.success(`Loaded ${PluginRegistry.getProviderIds().length} providers from MongoDB.`); + logger.info(`Configured channels: ${PluginRegistry.getChannels().join(', ')}`); +} + +/** + * Get list of configured channels from MongoDB (falls back to YAML if DB has no routings) + */ +export async function getConfiguredChannelsFromDatabase(): Promise { + try { + const routings = await ChannelRouting.find().lean(); + if (routings && routings.length > 0) { + return routings.map((r) => r.channel); + } + } catch (err) { + logger.warn('Could not query ChannelRouting from database:', { + error: err instanceof Error ? err.message : String(err) + }); + } + return getConfiguredChannels(); +} diff --git a/src/plugins/loader/plugin-fs.ts b/src/plugins/loader/plugin-fs.ts new file mode 100644 index 0000000..ba52a3c --- /dev/null +++ b/src/plugins/loader/plugin-fs.ts @@ -0,0 +1,191 @@ +/** + * Plugin File System & Package Utilities + * + * Manages local .plugins directory, dynamic imports, manifest extraction, + * and npm package installation/uninstallation. + */ + +import { readFileSync, existsSync, mkdirSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { execSync } from 'child_process'; +import { pathToFileURL } from 'url'; +import type { SimpleNSProvider, ProviderManifest } from '../interfaces/provider.types.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +export const PLUGINS_DIR = join(process.cwd(), '.plugins'); +export const PLUGINS_NODE_MODULES = join(PLUGINS_DIR, 'node_modules'); + +/** + * Initialize plugins directory with package.json if needed + */ +export function initPluginsDir(): void { + if (!existsSync(PLUGINS_DIR)) { + mkdirSync(PLUGINS_DIR, { recursive: true }); + } + + const packageJsonPath = join(PLUGINS_DIR, 'package.json'); + if (!existsSync(packageJsonPath)) { + const initialPackage = { + name: 'simplens-plugins', + version: '1.0.0', + description: 'User-installed SimpleNS plugins', + private: true, + type: 'module', + dependencies: {}, + }; + writeFileSync(packageJsonPath, JSON.stringify(initialPackage, null, 2)); + } +} + +/** + * Resolve the entry file for a package from its package.json + */ +export function resolvePackageEntry(packagePath: string): string { + const pkgJsonPath = join(packagePath, 'package.json'); + if (!existsSync(pkgJsonPath)) { + return join(packagePath, 'index.js'); + } + + const pkg = JSON.parse(readFileSync(pkgJsonPath, 'utf-8')); + + let entryPoint = 'index.js'; + if (pkg.exports) { + if (typeof pkg.exports === 'string') { + entryPoint = pkg.exports; + } else if (pkg.exports['.']) { + const dotExport = pkg.exports['.']; + entryPoint = + typeof dotExport === 'string' + ? dotExport + : dotExport.import || dotExport.default || 'index.js'; + } + } else if (pkg.main) { + entryPoint = pkg.main; + } + + return join(packagePath, entryPoint); +} + +/** + * Helper to instantiate provider from imported module + */ +export function instantiateFromModule( + module: Record, + packageName: string +): SimpleNSProvider { + if (module.default) { + if (typeof module.default === 'function') { + const DefaultExport = module.default as new () => SimpleNSProvider; + if (DefaultExport.prototype?.constructor) { + return new DefaultExport(); + } + return (module.default as () => SimpleNSProvider)(); + } + return module.default as SimpleNSProvider; + } + if (module.createProvider) { + return (module.createProvider as () => SimpleNSProvider)(); + } + throw new Error(`Package ${packageName} does not export a provider`); +} + +/** + * Dynamically import a provider package and instantiate + */ +export async function importAndInstantiateProvider(packageName: string): Promise { + try { + const pluginPath = join(PLUGINS_NODE_MODULES, packageName); + if (existsSync(pluginPath)) { + logger.debug(`Loading from plugins directory: ${packageName}`); + const entryFile = resolvePackageEntry(pluginPath); + const pluginUrl = pathToFileURL(entryFile).href; + const module = (await import(pluginUrl)) as Record; + return instantiateFromModule(module, packageName); + } + + // Fall back to core node_modules (for bundled plugins or testing) + const module = (await import(packageName)) as Record; + return instantiateFromModule(module, packageName); + } catch (_err) { + if (packageName.startsWith('./') || packageName.startsWith('../')) { + const module = (await import(packageName)) as Record; + return instantiateFromModule(module, packageName); + } + const wrappedErr = new Error(`Plugin not found: ${packageName}. Install with npm or dashboard.`); + (wrappedErr as unknown as Record).cause = _err; + throw wrappedErr; + } +} + +/** + * Extract manifest and authoritative package.json version from an installed package + */ +export async function extractPackageManifest( + packageName: string +): Promise<{ manifest: ProviderManifest; version: string }> { + let packageDir = join(PLUGINS_NODE_MODULES, packageName); + if (!existsSync(packageDir)) { + // Fall back to root node_modules if present + packageDir = join(process.cwd(), 'node_modules', packageName); + } + + const pkgJsonPath = join(packageDir, 'package.json'); + if (!existsSync(pkgJsonPath)) { + throw new Error(`package.json not found for installed plugin '${packageName}'`); + } + + const pkgJson = JSON.parse(readFileSync(pkgJsonPath, 'utf-8')); + const version: string = pkgJson.version || '1.0.0'; + + const provider = await importAndInstantiateProvider(packageName); + if (!provider.manifest) { + throw new Error(`Plugin '${packageName}' does not export a valid manifest property.`); + } + + // Ensure manifest version matches package.json + const manifest: ProviderManifest = { + ...provider.manifest, + version, + }; + + return { manifest, version }; +} + +/** + * Install an npm package into .plugins + */ +export function installNpmPackage(packageName: string, version?: string): void { + initPluginsDir(); + const specifier = version && version !== 'latest' ? `${packageName}@${version}` : `${packageName}@latest`; + logger.info(`Running npm install ${specifier} in ${PLUGINS_DIR}...`); + + try { + execSync(`npm install ${specifier}`, { + cwd: PLUGINS_DIR, + stdio: 'pipe', + }); + logger.success(`Successfully installed ${specifier}`); + } catch (err) { + logger.error(`Failed to install ${specifier}:`, err); + throw new Error(`Failed to install package: ${specifier}`, { cause: err }); + } +} + +/** + * Uninstall an npm package from .plugins + */ +export function uninstallNpmPackage(packageName: string): void { + initPluginsDir(); + logger.info(`Running npm uninstall ${packageName} in ${PLUGINS_DIR}...`); + + try { + execSync(`npm uninstall ${packageName}`, { + cwd: PLUGINS_DIR, + stdio: 'pipe', + }); + logger.success(`Successfully uninstalled ${packageName}`); + } catch (err) { + logger.error(`Failed to uninstall ${packageName}:`, err); + throw new Error(`Failed to uninstall package: ${packageName}`, { cause: err }); + } +} diff --git a/src/plugins/loader/registry.ts b/src/plugins/loader/registry.ts index e056f71..ff40753 100644 --- a/src/plugins/loader/registry.ts +++ b/src/plugins/loader/registry.ts @@ -22,7 +22,7 @@ export interface RegisteredProvider { */ export interface ChannelConfig { default: string; - fallback?: string; + fallback?: string | string[]; } /** @@ -54,7 +54,7 @@ export interface ProviderMetadata { export interface ChannelMetadata { providers: ProviderMetadata[]; default?: string; - fallback?: string; + fallback?: string | string[]; } /** @@ -156,6 +156,54 @@ class PluginRegistryClass { logger.info(`Registered provider: ${id} (${provider.manifest.displayName}) for channel: ${channel}`); } + /** + * Register or replace an existing provider + */ + registerOrReplace(provider: SimpleNSProvider, id: string, priority: number = 0): void { + if (this.providers.has(id)) { + this.unregister(id); + } + this.register(provider, id, priority); + } + + /** + * Unregister a provider by ID + */ + unregister(id: string): boolean { + const registered = this.providers.get(id); + if (!registered) { + return false; + } + + const channel = registered.provider.manifest.channel; + this.providers.delete(id); + + // Remove from channelProviders list + const existing = this.channelProviders.get(channel) || []; + const filtered = existing.filter(providerId => providerId !== id); + if (filtered.length > 0) { + this.channelProviders.set(channel, filtered); + } else { + this.channelProviders.delete(channel); + } + + // Clean up default / fallback if this provider was configured + const config = this.channelConfig.get(channel); + if (config) { + if (config.default === id) { + config.default = filtered[0] || ''; + } + if (Array.isArray(config.fallback)) { + config.fallback = config.fallback.filter(fId => fId !== id); + } else if (config.fallback === id) { + config.fallback = filtered[1] || undefined; + } + } + + logger.info(`Unregistered provider: ${id} from channel: ${channel}`); + return true; + } + /** * Set channel configuration (default/fallback) */ @@ -218,29 +266,46 @@ class PluginRegistryClass { } /** - * Get fallback provider for a channel + * Get ordered fallback provider IDs for a channel (cascading fallback array) */ - getFallbackProvider(channel: string): SimpleNSProvider | undefined { + getFallbackProviderIds(channel: string): string[] { const config = this.channelConfig.get(channel); if (config?.fallback) { - return this.get(config.fallback); + if (Array.isArray(config.fallback)) { + return config.fallback; + } + return [config.fallback]; } - // Fall back to second registered provider for channel - const providers = this.getProvidersForChannel(channel); - return providers[1]; + + // Default: return registered provider IDs for channel excluding default provider + const defaultId = this.getDefaultProviderId(channel); + const ids = this.channelProviders.get(channel) || []; + return ids.filter(id => id !== defaultId); + } + + /** + * Get ordered fallback providers for a channel + */ + getFallbackProviders(channel: string): SimpleNSProvider[] { + return this.getFallbackProviderIds(channel) + .map(id => this.get(id)) + .filter((p): p is SimpleNSProvider => p !== undefined); } /** - * Get fallback provider ID for a channel + * Get first fallback provider for a channel + */ + getFallbackProvider(channel: string): SimpleNSProvider | undefined { + const providers = this.getFallbackProviders(channel); + return providers[0]; + } + + /** + * Get first fallback provider ID for a channel */ getFallbackProviderId(channel: string): string | undefined { - const config = this.channelConfig.get(channel); - if (config?.fallback) { - return config.fallback; - } - // Fall back to second registered provider ID for channel - const ids = this.channelProviders.get(channel); - return ids?.[1]; + const ids = this.getFallbackProviderIds(channel); + return ids[0]; } /** diff --git a/src/plugins/loader/router.ts b/src/plugins/loader/router.ts index c7823ba..e09b8f5 100644 --- a/src/plugins/loader/router.ts +++ b/src/plugins/loader/router.ts @@ -11,7 +11,8 @@ import { handleSchemaValidationFailure } from '@src/processors/shared/schema-fai import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; /** - * Try fallback provider when primary fails with non-retryable error + * Try fallback providers when primary fails with non-retryable error. + * Cascades through fallback providers in sequence until one succeeds or all fail. */ async function tryFallback( channel: string, @@ -19,80 +20,101 @@ async function tryFallback( primaryError: DeliveryResult['error'], currentProviderId?: string ): Promise { - const fallbackProviderId = PluginRegistry.getFallbackProviderId(channel); - if (!fallbackProviderId) { - logger.debug(`[ProviderRouter] No fallback provider for ${channel}, returning error`); + const fallbackIds = PluginRegistry.getFallbackProviderIds(channel); + if (!fallbackIds || fallbackIds.length === 0) { + logger.debug(`[ProviderRouter] No fallback providers configured for ${channel}`); return null; } - if (currentProviderId && fallbackProviderId === currentProviderId) { - logger.debug(`[ProviderRouter] Fallback provider is same as current provider (${currentProviderId}), skipping fallback`); + const candidateIds = fallbackIds.filter(id => id !== currentProviderId); + if (candidateIds.length === 0) { + logger.debug(`[ProviderRouter] Fallback providers match current provider (${currentProviderId}), skipping fallback`); return null; } - const fallbackProvider = PluginRegistry.getFallbackProvider(channel); + let lastError = primaryError; - if (!fallbackProvider) { - logger.debug(`[ProviderRouter] No fallback provider for ${channel}, returning error`); - return null; - } - - logger.debug(`[ProviderRouter] Primary provider failed, trying fallback: ${fallbackProvider.manifest.name}`); + for (let i = 0; i < candidateIds.length; i++) { + const fallbackId = candidateIds[i]; + const fallbackProvider = PluginRegistry.get(fallbackId); + if (!fallbackProvider) { + logger.warn(`[ProviderRouter] Fallback provider '${fallbackId}' not found in registry, continuing cascade`); + continue; + } - void AdminAlertService.sendAlert('service_health', - `ℹ️ USING FALLBACK PROVIDER\n` + - `Channel: ${channel}\n` + - `Notification ID: ${notification.notification_id}\n` + - `Primary provider error: ${primaryError?.message}\n` + - `Fallback provider: ${fallbackProvider.manifest.displayName}\n` + - `Action: Check primary provider configuration if this occurs frequently.`, - { severity: 'info', notificationId: notification.notification_id, channel }); - - const schema = fallbackProvider.getNotificationSchema(); - const validationResult = schema.safeParse(notification); - - if (!validationResult.success) { - logger.error(`[${channel}] Invalid notification schema for fallback provider of channel:${channel} and provider name ${fallbackProvider.manifest.displayName}:`, - validationResult.error.issues); - - await handleSchemaValidationFailure( - notification.notification_id, - channel, - notification, - validationResult.error, - 'fallback provider' - ); - return { - success: false, - error: { + logger.info(`[ProviderRouter] Trying fallback provider [${i + 1}/${candidateIds.length}]: ${fallbackId} (${fallbackProvider.manifest.displayName})`); + + void AdminAlertService.sendAlert('service_health', + `ℹ️ USING FALLBACK PROVIDER (${i + 1}/${candidateIds.length})\n` + + `Channel: ${channel}\n` + + `Notification ID: ${notification.notification_id}\n` + + `Previous error: ${lastError?.message}\n` + + `Fallback provider: ${fallbackProvider.manifest.displayName} (${fallbackId})\n` + + `Action: Check provider configuration if this occurs frequently.`, + { severity: 'info', notificationId: notification.notification_id, channel }); + + const schema = fallbackProvider.getNotificationSchema(); + const validationResult = schema.safeParse(notification); + + if (!validationResult.success) { + logger.error(`[${channel}] Invalid notification schema for fallback provider '${fallbackId}':`, + validationResult.error.issues); + + await handleSchemaValidationFailure( + notification.notification_id, + channel, + notification, + validationResult.error, + `fallback provider ${fallbackId}` + ); + lastError = { code: 'FALLBACK_SCHEMA_VALIDATION_ERROR', - message: 'Fallback provider schema validation failed', + message: `Fallback provider ${fallbackId} schema validation failed`, retryable: false - } - }; - } + }; + continue; + } - const fallbackResult = await fallbackProvider.send(notification); + try { + const fallbackResult = await fallbackProvider.send(notification); + if (fallbackResult.success) { + logger.success(`[${channel}] Notification delivered via fallback provider: ${fallbackId}`); + return { + ...fallbackResult, + messageId: fallbackResult.messageId || `fallback-${fallbackId}`, + }; + } - if (fallbackResult.success) { - return fallbackResult; + lastError = fallbackResult.error || { + code: 'FALLBACK_SEND_FAILED', + message: `Fallback provider ${fallbackId} send failed without error details`, + retryable: false + }; + } catch (sendErr) { + logger.error(`[${channel}] Error sending via fallback provider ${fallbackId}:`, sendErr); + lastError = { + code: 'FALLBACK_EXECUTION_ERROR', + message: sendErr instanceof Error ? sendErr.message : String(sendErr), + retryable: false + }; + } } - // Both failed - return last error + // All fallbacks failed in cascade void AdminAlertService.sendAlert('failed_notification', - `🔴 ALL PROVIDERS FAILED\n` + + `🔴 ALL PROVIDERS IN CASCADE FAILED\n` + `Channel: ${channel}\n` + `Notification ID: ${notification.notification_id}\n` + - `Primary provider error: ${primaryError?.message}\n` + - `Fallback error: ${fallbackResult.error?.message}\n` + - `Action: Verify provider credentials. Check SMTP/API connectivity. Review simplens.config.yaml.`, + `Attempted Fallbacks: ${candidateIds.join(', ')}\n` + + `Last error: ${lastError?.message}\n` + + `Action: Verify all provider credentials and connectivity for channel ${channel}.`, { severity: 'critical', notificationId: notification.notification_id, channel }); return { success: false, error: { code: 'ALL_PROVIDERS_FAILED', - message: `All providers failed. Last error: ${fallbackResult.error?.message || 'Unknown'}`, + message: `All providers in cascade failed. Last error: ${lastError?.message || 'Unknown'}`, retryable: false, }, }; @@ -106,13 +128,22 @@ export function resolveFallbackProviderId( channel: string, currentProviderId?: string ): string | undefined { - const fallbackProviderId = PluginRegistry.getFallbackProviderId(channel); + const fallbackIds = PluginRegistry.getFallbackProviderIds(channel); + if (!fallbackIds || fallbackIds.length === 0) return undefined; + if (!currentProviderId) return fallbackIds[0]; + + // If currentProviderId is in the cascade, advance to the next fallback provider + const currentIndex = fallbackIds.indexOf(currentProviderId); + if (currentIndex >= 0 && currentIndex + 1 < fallbackIds.length) { + return fallbackIds[currentIndex + 1]; + } - if (!fallbackProviderId || fallbackProviderId === currentProviderId) { - return undefined; + // If current was primary, return first fallback + if (currentIndex === -1) { + return fallbackIds[0]; } - return fallbackProviderId; + return undefined; } /** diff --git a/src/plugins/services/channel-routing.service.ts b/src/plugins/services/channel-routing.service.ts new file mode 100644 index 0000000..0d25997 --- /dev/null +++ b/src/plugins/services/channel-routing.service.ts @@ -0,0 +1,153 @@ +/** + * Channel Routing Service + * + * Manages channel default/cascading fallback provider configuration, + * dynamic Kafka partition scaling, and Redis pub/sub synchronization. + */ + +import ChannelRouting from '@src/database/models/channel-routing.models.js'; +import Provider from '@src/database/models/provider.models.js'; +import { PluginRegistry } from '@src/plugins/loader/registry.js'; +import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; +import { expandTopicPartitions } from '@src/config/kafka.config.js'; +import type { channel_routing_document } from '@src/types/types.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +export class ChannelRoutingService { + /** + * List all configured channel routings + */ + public static async listChannelRoutings(): Promise { + return (await ChannelRouting.find().sort({ channel: 1 }).lean()) as channel_routing_document[]; + } + + /** + * Get routing configuration for a specific channel + */ + public static async getChannelRouting(channel: string): Promise { + return (await ChannelRouting.findOne({ channel }).lean()) as channel_routing_document | null; + } + + /** + * Set or update channel routing configuration and expand Kafka partitions if requested + */ + public static async setChannelRouting( + channel: string, + data: { + default_provider_id: string; + fallback_provider_ids?: string[]; + partitions?: number; + } + ): Promise { + logger.info(`Updating channel routing for '${channel}'...`); + + // 1. Validate default provider + const defaultProvider = await Provider.findOne({ id: data.default_provider_id }); + if (!defaultProvider) { + throw new Error(`Default provider '${data.default_provider_id}' not found.`); + } + if (defaultProvider.channel !== channel) { + throw new Error( + `Default provider '${data.default_provider_id}' is configured for channel '${defaultProvider.channel}', not '${channel}'.` + ); + } + + // 2. Validate fallback providers + const fallbacks = data.fallback_provider_ids || []; + for (const fId of fallbacks) { + const fProvider = await Provider.findOne({ id: fId }); + if (!fProvider) { + throw new Error(`Fallback provider '${fId}' not found.`); + } + if (fProvider.channel !== channel) { + throw new Error( + `Fallback provider '${fId}' is configured for channel '${fProvider.channel}', not '${channel}'.` + ); + } + } + + // 3. Handle dynamic Kafka partition scaling + const existing = await ChannelRouting.findOne({ channel }); + const currentPartitions = existing?.partitions || 6; + const desiredPartitions = data.partitions !== undefined ? data.partitions : currentPartitions; + + if (desiredPartitions < currentPartitions) { + throw new Error( + `Cannot decrease partitions for channel '${channel}' from ${currentPartitions} to ${desiredPartitions}. Kafka partitions can only be increased.` + ); + } + + if (desiredPartitions > currentPartitions) { + logger.info( + `Requesting Kafka partition expansion for channel '${channel}' from ${currentPartitions} to ${desiredPartitions}...` + ); + try { + await expandTopicPartitions(channel, desiredPartitions); + } catch (kafkaErr) { + logger.error(`Failed to expand Kafka topic partitions for channel '${channel}':`, kafkaErr); + throw new Error(`Failed to expand Kafka partitions: ${(kafkaErr as Error).message}`, { + cause: kafkaErr, + }); + } + } + + // 4. Save to MongoDB + const routing = await ChannelRouting.findOneAndUpdate( + { channel }, + { + channel, + default_provider_id: data.default_provider_id, + fallback_provider_ids: fallbacks, + partitions: desiredPartitions, + }, + { upsert: true, new: true } + ); + + // 5. Update local PluginRegistry + PluginRegistry.setChannelConfig(channel, { + default: data.default_provider_id, + fallback: fallbacks, + }); + + // 6. Broadcast sync event + await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { + channel, + }); + + logger.success(`Channel routing for '${channel}' updated successfully.`); + return routing.toObject() as channel_routing_document; + } + + /** + * Delete routing configuration for a channel + */ + public static async deleteChannelRouting(channel: string): Promise { + logger.info(`Deleting channel routing for '${channel}'...`); + await ChannelRouting.deleteOne({ channel }); + + await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { + channel, + }); + logger.success(`Channel routing for '${channel}' deleted.`); + } + + /** + * Register remote synchronization handlers with Redis Pub/Sub + */ + public static registerSyncHandlers(): void { + PluginSyncService.on('CHANNEL_ROUTING_UPDATED', async (msg) => { + const { channel } = msg.payload; + if (!channel) return; + + logger.info(`Sync: Handling remote CHANNEL_ROUTING_UPDATED for '${channel}'...`); + const routing = await ChannelRouting.findOne({ channel }); + + if (routing) { + PluginRegistry.setChannelConfig(channel, { + default: routing.default_provider_id, + fallback: routing.fallback_provider_ids || [], + }); + } + }); + } +} diff --git a/src/plugins/services/plugin-manager.service.ts b/src/plugins/services/plugin-manager.service.ts new file mode 100644 index 0000000..aa72245 --- /dev/null +++ b/src/plugins/services/plugin-manager.service.ts @@ -0,0 +1,196 @@ +/** + * Plugin Manager Service + * + * Handles npm package installation, version changes, uninstallation, + * metadata extraction, database records, and Redis pub/sub synchronization. + */ + +import Plugin from '@src/database/models/plugin.models.js'; +import Provider from '@src/database/models/provider.models.js'; +import { + installNpmPackage, + uninstallNpmPackage, + extractPackageManifest, +} from '@src/plugins/loader/plugin-fs.js'; +import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; +import type { plugin_document } from '@src/types/types.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +export class PluginManagerService { + /** + * Install an npm package, extract its manifest, persist to MongoDB, and broadcast. + */ + public static async installPlugin( + packageName: string, + version?: string + ): Promise { + logger.info(`Installing plugin package '${packageName}' (version: ${version || 'latest'})...`); + + return await PluginSyncService.runWithLock(async () => { + try { + installNpmPackage(packageName, version); + const { manifest, version: authoritativeVersion } = await extractPackageManifest(packageName); + + const plugin = await Plugin.findOneAndUpdate( + { name: packageName }, + { + name: packageName, + version: authoritativeVersion, + status: 'installed', + manifest, + error: undefined, + }, + { upsert: true, new: true } + ); + + await PluginSyncService.publish('PLUGIN_INSTALLED', { + plugin_name: packageName, + version: authoritativeVersion, + }); + + logger.success(`Plugin '${packageName}'@${authoritativeVersion} installed successfully.`); + return plugin.toObject() as plugin_document; + } catch (err) { + logger.error(`Failed to install plugin '${packageName}':`, err); + throw err; + } + }); + } + + /** + * Change/upgrade/downgrade the version of an already installed plugin. + */ + public static async changePluginVersion( + packageName: string, + version: string + ): Promise { + logger.info(`Changing version of plugin '${packageName}' to '${version}'...`); + + const existing = await Plugin.findOne({ name: packageName }); + if (!existing) { + throw new Error(`Plugin '${packageName}' is not installed.`); + } + + return await PluginSyncService.runWithLock(async () => { + try { + installNpmPackage(packageName, version); + const { manifest, version: authoritativeVersion } = await extractPackageManifest(packageName); + + const plugin = await Plugin.findOneAndUpdate( + { name: packageName }, + { + version: authoritativeVersion, + status: 'installed', + manifest, + error: undefined, + }, + { new: true } + ); + + await PluginSyncService.publish('PLUGIN_VERSION_CHANGED', { + plugin_name: packageName, + version: authoritativeVersion, + }); + + logger.success(`Plugin '${packageName}' updated to version ${authoritativeVersion}.`); + return plugin!.toObject() as plugin_document; + } catch (err) { + logger.error(`Failed to update plugin '${packageName}' to version '${version}':`, err); + throw err; + } + }); + } + + /** + * Uninstall a plugin. Rejects if any configured providers depend on it. + */ + public static async uninstallPlugin(packageName: string): Promise { + logger.info(`Uninstalling plugin '${packageName}'...`); + + const dependentProvidersCount = await Provider.countDocuments({ plugin_name: packageName }); + if (dependentProvidersCount > 0) { + throw new Error( + `Cannot uninstall plugin '${packageName}': ${dependentProvidersCount} provider(s) depend on it. Delete or reassign providers first.` + ); + } + + const existing = await Plugin.findOne({ name: packageName }); + if (!existing) { + throw new Error(`Plugin '${packageName}' not found.`); + } + + await PluginSyncService.runWithLock(async () => { + try { + uninstallNpmPackage(packageName); + await Plugin.deleteOne({ name: packageName }); + + await PluginSyncService.publish('PLUGIN_UNINSTALLED', { + plugin_name: packageName, + }); + + logger.success(`Plugin '${packageName}' uninstalled successfully.`); + } catch (err) { + logger.error(`Failed to uninstall plugin '${packageName}':`, err); + throw err; + } + }); + } + + /** + * List all installed plugins. + */ + public static async listInstalledPlugins(): Promise { + return (await Plugin.find().sort({ name: 1 }).lean()) as plugin_document[]; + } + + /** + * Get a single installed plugin by name. + */ + public static async getPlugin(name: string): Promise { + return (await Plugin.findOne({ name }).lean()) as plugin_document | null; + } + + /** + * Register remote synchronization handlers with Redis Pub/Sub + */ + public static registerSyncHandlers(): void { + PluginSyncService.on('PLUGIN_INSTALLED', async (msg) => { + const { plugin_name, version } = msg.payload; + if (!plugin_name) return; + logger.info(`Sync: Installing remote plugin '${plugin_name}' (${version || 'latest'})...`); + await PluginSyncService.runWithLock(async () => { + try { + installNpmPackage(plugin_name, version); + } catch (err) { + logger.error(`Failed to sync remote install of '${plugin_name}':`, err); + } + }); + }); + + PluginSyncService.on('PLUGIN_VERSION_CHANGED', async (msg) => { + const { plugin_name, version } = msg.payload; + if (!plugin_name) return; + logger.info(`Sync: Changing remote plugin '${plugin_name}' to version '${version}'...`); + await PluginSyncService.runWithLock(async () => { + try { + installNpmPackage(plugin_name, version); + } catch (err) { + logger.error(`Failed to sync remote version change of '${plugin_name}':`, err); + } + }); + }); + + PluginSyncService.on('PLUGIN_UNINSTALLED', async (msg) => { + const { plugin_name } = msg.payload; + if (!plugin_name) return; + logger.info(`Sync: Uninstalling remote plugin '${plugin_name}'...`); + await PluginSyncService.runWithLock(async () => { + try { + uninstallNpmPackage(plugin_name); + } catch (err) { + logger.error(`Failed to sync remote uninstall of '${plugin_name}':`, err); + } + }); + }); + } +} diff --git a/src/plugins/services/provider-manager.service.ts b/src/plugins/services/provider-manager.service.ts new file mode 100644 index 0000000..9bad510 --- /dev/null +++ b/src/plugins/services/provider-manager.service.ts @@ -0,0 +1,365 @@ +/** + * Provider Manager Service + * + * Handles CRUD operations for provider instances, envelope encryption of credentials, + * live connection testing, registry loading, and Redis synchronization. + */ + +import Provider from '@src/database/models/provider.models.js'; +import Plugin from '@src/database/models/plugin.models.js'; +import ChannelRouting from '@src/database/models/channel-routing.models.js'; +import { + encryptCredentials, + decryptCredentials, +} from '@src/plugins/crypto/keypair-manager.js'; +import { importAndInstantiateProvider } from '@src/plugins/loader/plugin-fs.js'; +import { PluginRegistry } from '@src/plugins/loader/registry.js'; +import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; +import type { provider_document } from '@src/types/types.js'; +import type { ProviderConfig } from '@src/plugins/interfaces/provider.types.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +export interface ProviderResponseDto { + _id?: string; + id: string; + plugin_name: string; + channel: string; + priority: number; + enabled: boolean; + options?: Record; + credentials_configured: boolean; + created_at?: Date; + updated_at?: Date; +} + +export class ProviderManagerService { + /** + * Create a new provider instance with envelope-encrypted credentials + */ + public static async createProvider(data: { + id: string; + plugin_name: string; + credentials: Record; + priority?: number; + options?: Record; + enabled?: boolean; + }): Promise { + logger.info(`Creating provider '${data.id}' using plugin '${data.plugin_name}'...`); + + const existing = await Provider.findOne({ id: data.id }); + if (existing) { + throw new Error(`Provider with ID '${data.id}' already exists.`); + } + + const plugin = await Plugin.findOne({ name: data.plugin_name }); + if (!plugin) { + throw new Error(`Plugin '${data.plugin_name}' is not installed.`); + } + + const encryptedCredentials = await encryptCredentials(data.credentials); + + const providerDoc = await Provider.create({ + id: data.id, + plugin_name: data.plugin_name, + channel: plugin.manifest.channel, + priority: data.priority ?? 0, + enabled: data.enabled !== false, + credentials: encryptedCredentials, + options: data.options || {}, + }); + + // If enabled, load into current process registry + if (providerDoc.enabled) { + try { + await ProviderManagerService.loadAndRegisterProvider(providerDoc.toObject() as provider_document); + } catch (err) { + logger.warn(`Provider '${data.id}' registered in DB, but failed local initialization:`, { + error: err instanceof Error ? err.message : String(err), + }); + } + } + + await PluginSyncService.publish('PROVIDER_UPSERTED', { + provider_id: data.id, + }); + + logger.success(`Provider '${data.id}' created successfully.`); + + return { + _id: providerDoc._id.toString(), + id: providerDoc.id, + plugin_name: providerDoc.plugin_name, + channel: providerDoc.channel, + priority: providerDoc.priority, + enabled: providerDoc.enabled, + options: providerDoc.options as Record | undefined, + credentials_configured: true, + created_at: providerDoc.created_at, + updated_at: providerDoc.updated_at, + }; + } + + /** + * Update an existing provider instance + */ + public static async updateProvider( + id: string, + updates: { + credentials?: Record; + priority?: number; + options?: Record; + enabled?: boolean; + } + ): Promise { + logger.info(`Updating provider '${id}'...`); + + const providerDoc = await Provider.findOne({ id }); + if (!providerDoc) { + throw new Error(`Provider '${id}' not found.`); + } + + if (updates.priority !== undefined) { + providerDoc.priority = updates.priority; + } + if (updates.enabled !== undefined) { + providerDoc.enabled = updates.enabled; + } + if (updates.options !== undefined) { + providerDoc.options = updates.options; + } + if (updates.credentials && Object.keys(updates.credentials).length > 0) { + providerDoc.credentials = await encryptCredentials(updates.credentials); + } + + await providerDoc.save(); + + if (providerDoc.enabled) { + try { + await ProviderManagerService.loadAndRegisterProvider(providerDoc.toObject() as provider_document); + } catch (err) { + logger.warn(`Provider '${id}' updated in DB, but failed local reload:`, { + error: err instanceof Error ? err.message : String(err), + }); + } + } else { + PluginRegistry.unregister(id); + } + + await PluginSyncService.publish('PROVIDER_UPSERTED', { + provider_id: id, + }); + + logger.success(`Provider '${id}' updated successfully.`); + + return { + _id: providerDoc._id.toString(), + id: providerDoc.id, + plugin_name: providerDoc.plugin_name, + channel: providerDoc.channel, + priority: providerDoc.priority, + enabled: providerDoc.enabled, + options: providerDoc.options as Record | undefined, + credentials_configured: true, + created_at: providerDoc.created_at, + updated_at: providerDoc.updated_at, + }; + } + + /** + * Delete a provider instance. Rejects if used in channel routing. + */ + public static async deleteProvider(id: string): Promise { + logger.info(`Deleting provider '${id}'...`); + + const routingUse = await ChannelRouting.findOne({ + $or: [{ default_provider_id: id }, { fallback_provider_ids: id }], + }); + + if (routingUse) { + throw new Error( + `Cannot delete provider '${id}': it is configured in channel '${routingUse.channel}' routing.` + ); + } + + const providerDoc = await Provider.findOne({ id }); + if (!providerDoc) { + throw new Error(`Provider '${id}' not found.`); + } + + await Provider.deleteOne({ id }); + PluginRegistry.unregister(id); + + await PluginSyncService.publish('PROVIDER_DELETED', { + provider_id: id, + }); + + logger.success(`Provider '${id}' deleted successfully.`); + } + + /** + * List all providers without exposing ciphertext credentials + */ + public static async listProviders(): Promise { + const providers = await Provider.find().sort({ channel: 1, priority: -1 }).lean(); + + return providers.map((p) => ({ + _id: p._id.toString(), + id: p.id, + plugin_name: p.plugin_name, + channel: p.channel, + priority: p.priority, + enabled: p.enabled, + options: p.options as Record | undefined, + credentials_configured: !!p.credentials?.encrypted_data, + created_at: p.created_at, + updated_at: p.updated_at, + })); + } + + /** + * Get single provider by ID + */ + public static async getProvider( + id: string, + includeDecrypted = false + ): Promise<(ProviderResponseDto & { decrypted_credentials?: Record }) | null> { + const providerDoc = await Provider.findOne({ id }).lean(); + if (!providerDoc) return null; + + let decrypted: Record | undefined; + if (includeDecrypted && providerDoc.credentials) { + decrypted = await decryptCredentials(providerDoc.credentials); + } + + return { + _id: providerDoc._id.toString(), + id: providerDoc.id, + plugin_name: providerDoc.plugin_name, + channel: providerDoc.channel, + priority: providerDoc.priority, + enabled: providerDoc.enabled, + options: providerDoc.options as Record | undefined, + credentials_configured: !!providerDoc.credentials?.encrypted_data, + decrypted_credentials: decrypted, + created_at: providerDoc.created_at, + updated_at: providerDoc.updated_at, + }; + } + + /** + * Test live provider connection / credentials + */ + public static async testProviderConnection(data: { + provider_id?: string; + plugin_name?: string; + credentials?: Record; + options?: Record; + }): Promise<{ success: boolean; message: string }> { + let pluginName = data.plugin_name; + let credentials = data.credentials; + let options = data.options || {}; + + if (data.provider_id) { + const existing = await Provider.findOne({ id: data.provider_id }); + if (!existing) { + throw new Error(`Provider '${data.provider_id}' not found.`); + } + pluginName = existing.plugin_name; + options = { ...(existing.options as Record), ...options }; + if (!credentials || Object.keys(credentials).length === 0) { + credentials = await decryptCredentials(existing.credentials); + } + } + + if (!pluginName) { + throw new Error('plugin_name or provider_id is required.'); + } + if (!credentials) { + throw new Error('Provider credentials are required.'); + } + + try { + const providerInstance = await importAndInstantiateProvider(pluginName); + const config: ProviderConfig = { + id: data.provider_id || 'test-connection', + credentials, + options, + }; + + await providerInstance.initialize(config); + const healthy = await providerInstance.healthCheck(); + + if (healthy) { + return { success: true, message: 'Provider connection and health check succeeded.' }; + } else { + return { success: false, message: 'Provider health check failed.' }; + } + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + return { success: false, message: `Connection test error: ${message}` }; + } + } + + /** + * Load, decrypt credentials, instantiate and register provider into PluginRegistry + */ + public static async loadAndRegisterProvider( + providerDoc: provider_document, + initialize = true + ): Promise { + const credentials = await decryptCredentials(providerDoc.credentials); + const providerInstance = await importAndInstantiateProvider(providerDoc.plugin_name); + + if (initialize) { + const config: ProviderConfig = { + id: providerDoc.id, + credentials, + options: { + ...(providerDoc.options as Record), + priority: providerDoc.priority, + }, + }; + + await providerInstance.initialize(config); + + const healthy = await providerInstance.healthCheck(); + if (!healthy) { + logger.warn(`Provider '${providerDoc.id}' failed health check on initial load.`); + } + } + + PluginRegistry.registerOrReplace(providerInstance, providerDoc.id, providerDoc.priority ?? 0); + logger.success(`Registered provider '${providerDoc.id}' in PluginRegistry.`); + } + + /** + * Register remote synchronization handlers with Redis Pub/Sub + */ + public static registerSyncHandlers(): void { + PluginSyncService.on('PROVIDER_UPSERTED', async (msg) => { + const { provider_id } = msg.payload; + if (!provider_id) return; + + logger.info(`Sync: Handling remote PROVIDER_UPSERTED for '${provider_id}'...`); + const providerDoc = await Provider.findOne({ id: provider_id }); + + if (providerDoc && providerDoc.enabled) { + try { + await ProviderManagerService.loadAndRegisterProvider(providerDoc.toObject() as provider_document); + } catch (err) { + logger.error(`Failed to reload synced provider '${provider_id}':`, err); + } + } else { + PluginRegistry.unregister(provider_id); + } + }); + + PluginSyncService.on('PROVIDER_DELETED', async (msg) => { + const { provider_id } = msg.payload; + if (!provider_id) return; + + logger.info(`Sync: Handling remote PROVIDER_DELETED for '${provider_id}'...`); + PluginRegistry.unregister(provider_id); + }); + } +} diff --git a/src/plugins/sync/plugin-sync.service.ts b/src/plugins/sync/plugin-sync.service.ts new file mode 100644 index 0000000..74a60d6 --- /dev/null +++ b/src/plugins/sync/plugin-sync.service.ts @@ -0,0 +1,194 @@ +/** + * Plugin System Redis Pub/Sub Synchronization Service + * + * Synchronizes plugin installations, provider changes, and channel routing + * across all running API and notification_processor containers in real time. + */ + +import { randomUUID } from 'crypto'; +import type { Redis as RedisType } from 'ioredis'; +import { getRedisClient } from '@src/config/redis.config.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +export const PLUGIN_SYNC_CHANNEL = 'simplens:plugin_system:sync'; + +export type PluginSyncAction = + | 'PLUGIN_INSTALLED' + | 'PLUGIN_UNINSTALLED' + | 'PLUGIN_VERSION_CHANGED' + | 'PROVIDER_UPSERTED' + | 'PROVIDER_DELETED' + | 'CHANNEL_ROUTING_UPDATED' + | 'SYSTEM_RELOAD'; + +export interface PluginSyncPayload { + plugin_name?: string; + version?: string; + provider_id?: string; + channel?: string; + [key: string]: unknown; +} + +export interface PluginSyncMessage { + event_id: string; + source_instance_id: string; + timestamp: string; + action: PluginSyncAction; + payload: PluginSyncPayload; +} + +export type PluginSyncHandler = (message: PluginSyncMessage) => Promise; + +export class PluginSyncServiceClass { + private static instance: PluginSyncServiceClass; + private instanceId: string; + private subClient: RedisType | null = null; + private isSubscribed = false; + private handlers: Map = new Map(); + private installMutex: Promise = Promise.resolve(); + + private constructor() { + this.instanceId = process.env.HOSTNAME || process.env.WORKER_ID || `instance-${randomUUID().slice(0, 8)}`; + } + + public static getInstance(): PluginSyncServiceClass { + if (!PluginSyncServiceClass.instance) { + PluginSyncServiceClass.instance = new PluginSyncServiceClass(); + } + return PluginSyncServiceClass.instance; + } + + public getInstanceId(): string { + return this.instanceId; + } + + /** + * Run an asynchronous task with in-process mutex locking + * Useful for protecting the .plugins/ folder during npm install/uninstall + */ + public async runWithLock(task: () => Promise): Promise { + let release: () => void = () => {}; + const nextLock = new Promise((resolve) => { + release = resolve; + }); + + const currentLock = this.installMutex; + this.installMutex = nextLock; + + await currentLock; + try { + return await task(); + } finally { + release(); + } + } + + /** + * Register an event handler for a specific action (or '*' for all actions) + */ + public on(action: PluginSyncAction | '*', handler: PluginSyncHandler): void { + const list = this.handlers.get(action) || []; + list.push(handler); + this.handlers.set(action, list); + } + + /** + * Publish an event to the Redis synchronization channel + */ + public async publish(action: PluginSyncAction, payload: PluginSyncPayload = {}): Promise { + try { + const redis = getRedisClient(); + const message: PluginSyncMessage = { + event_id: randomUUID(), + source_instance_id: this.instanceId, + timestamp: new Date().toISOString(), + action, + payload, + }; + + await redis.publish(PLUGIN_SYNC_CHANNEL, JSON.stringify(message)); + logger.info(`Published sync event [${action}] to ${PLUGIN_SYNC_CHANNEL}`); + } catch (err) { + logger.error(`Failed to publish sync event [${action}] to Redis`, err); + } + } + + /** + * Initialize and start the Redis subscriber + */ + public async startSubscriber(): Promise { + if (this.isSubscribed) { + return; + } + + try { + const baseClient = getRedisClient(); + this.subClient = baseClient.duplicate(); + + await this.subClient.connect(); + await this.subClient.subscribe(PLUGIN_SYNC_CHANNEL); + this.isSubscribed = true; + + this.subClient.on('message', async (channel, messageStr) => { + if (channel !== PLUGIN_SYNC_CHANNEL) return; + + try { + const message: PluginSyncMessage = JSON.parse(messageStr); + + // Skip events emitted by this instance to prevent loops + if (message.source_instance_id === this.instanceId) { + logger.debug(`Ignoring sync event [${message.action}] from self (${this.instanceId})`); + return; + } + + logger.info(`Received sync event [${message.action}] from ${message.source_instance_id}`); + + // Execute action-specific handlers + const specificHandlers = this.handlers.get(message.action) || []; + for (const handler of specificHandlers) { + try { + await handler(message); + } catch (handlerErr) { + logger.error(`Error in sync handler for [${message.action}]:`, handlerErr); + } + } + + // Execute wildcard handlers + const wildcardHandlers = this.handlers.get('*') || []; + for (const handler of wildcardHandlers) { + try { + await handler(message); + } catch (handlerErr) { + logger.error(`Error in wildcard sync handler:`, handlerErr); + } + } + } catch (parseErr) { + logger.error('Failed to parse sync message from Redis:', parseErr); + } + }); + + logger.success(`Subscribed to plugin sync channel: ${PLUGIN_SYNC_CHANNEL}`); + } catch (err) { + logger.error('Failed to initialize Redis subscriber for plugin sync:', err); + } + } + + /** + * Disconnect the subscriber + */ + public async stopSubscriber(): Promise { + if (this.subClient) { + try { + await this.subClient.unsubscribe(PLUGIN_SYNC_CHANNEL); + await this.subClient.quit(); + } catch (err) { + logger.error('Error stopping plugin sync subscriber:', err); + } finally { + this.subClient = null; + this.isSubscribed = false; + } + } + } +} + +export const PluginSyncService = PluginSyncServiceClass.getInstance(); diff --git a/src/processors/unified/unified.consumer.ts b/src/processors/unified/unified.consumer.ts index 42f3be4..b96cdcb 100644 --- a/src/processors/unified/unified.consumer.ts +++ b/src/processors/unified/unified.consumer.ts @@ -526,3 +526,10 @@ export const stopAllConsumers = async (): Promise => { await stopUnifiedConsumer(channel); } }; + +/** + * Get list of currently running consumer channels + */ +export const getActiveConsumerChannels = (): string[] => { + return Array.from(consumers.keys()); +}; diff --git a/src/processors/unified/unified.processor.ts b/src/processors/unified/unified.processor.ts index bae6baa..97a0e01 100644 --- a/src/processors/unified/unified.processor.ts +++ b/src/processors/unified/unified.processor.ts @@ -1,204 +1,284 @@ /** * Unified Processor - Main entry point for plugin-based notification processing * - * Supports both single-channel and multi-channel modes: + * Supports dynamic MongoDB configuration, Redis Pub/Sub syncing, + * idle standby mode when empty, and both single-channel and multi-channel modes: * - Single: PROCESSOR_CHANNEL=email (or --channel email) * - Multi: PROCESSOR_CHANNEL=all (or omit to process all configured channels) */ +import { connectMongoDB } from '@src/config/db.config.js'; import { connectRedis, disconnectRedis } from '@src/config/redis.config.js'; import { initStatusProducer, disconnectStatusProducer } from '@src/processors/shared/status.producer.js'; import { initDelayedProducer, disconnectDelayedProducer } from '@src/processors/shared/delayed.producer.js'; -import { loadProvidersFromEnv, PluginRegistry } from '@src/plugins/index.js'; -import { startUnifiedConsumer, stopAllConsumers } from './unified.consumer.js'; +import { + PluginRegistry, + PluginSyncService, + PluginManagerService, + ProviderManagerService, + ChannelRoutingService, + loadProvidersFromDatabase, +} from '@src/plugins/index.js'; +import { + startUnifiedConsumer, + stopUnifiedConsumer, + stopAllConsumers, + getActiveConsumerChannels, +} from './unified.consumer.js'; import { unifiedProcessorLogger as logger } from './unified.logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; -//Import the admin channel provider files here for them to self-register +// Import the admin channel provider files here for them to self-register import "@src/admin-alerts/channels/discord.channel.js"; import "@src/admin-alerts/channels/telegram.channel.js"; // Track active channels for shutdown const activeChannels: string[] = []; let isShuttingDown = false; +let dbConnection: { disconnect: () => Promise } | null = null; /** * Parse channel argument from CLI or environment */ const getChannelConfig = (): string[] | 'all' => { - // Check CLI args first: --channel email - const channelArgIndex = process.argv.indexOf('--channel'); - if (channelArgIndex !== -1 && process.argv[channelArgIndex + 1]) { - const channel = process.argv[channelArgIndex + 1]; - return channel === 'all' ? 'all' : [channel]; - } + const channelArgIndex = process.argv.indexOf('--channel'); + if (channelArgIndex !== -1 && process.argv[channelArgIndex + 1]) { + const channel = process.argv[channelArgIndex + 1]; + return channel === 'all' ? 'all' : [channel]; + } + + const envChannel = process.env.PROCESSOR_CHANNEL; + if (envChannel) { + return envChannel === 'all' ? 'all' : [envChannel]; + } + + return 'all'; +}; - // Fall back to environment variable - const envChannel = process.env.PROCESSOR_CHANNEL; - if (envChannel) { - return envChannel === 'all' ? 'all' : [envChannel]; +/** + * Check for new channels and start consumers if configured + */ +const syncActiveConsumers = async (): Promise => { + if (isShuttingDown) return; + + const loadedChannels = PluginRegistry.getChannels(); + const currentRunning = getActiveConsumerChannels(); + const channelConfig = getChannelConfig(); + + for (const channel of loadedChannels) { + const shouldRun = + channelConfig === 'all' || (Array.isArray(channelConfig) && channelConfig.includes(channel)); + + if (shouldRun && !currentRunning.includes(channel)) { + try { + logger.info(`Starting dynamic consumer for new channel: ${channel}`); + await startUnifiedConsumer(channel); + if (!activeChannels.includes(channel)) { + activeChannels.push(channel); + } + } catch (err) { + logger.error(`Failed to dynamically start consumer for channel '${channel}':`, err); + } } - - // Default: process all channels - return 'all'; + } + + // If a channel was removed from PluginRegistry, stop its consumer + for (const runningChannel of currentRunning) { + if (!loadedChannels.includes(runningChannel)) { + logger.info(`Stopping consumer for removed channel: ${runningChannel}`); + await stopUnifiedConsumer(runningChannel); + const index = activeChannels.indexOf(runningChannel); + if (index !== -1) { + activeChannels.splice(index, 1); + } + } + } }; /** * Graceful shutdown handler */ const gracefulShutdown = async (signal: string): Promise => { - if (isShuttingDown) { - logger.info('Shutdown already in progress...'); - return; + if (isShuttingDown) { + logger.info('Shutdown already in progress...'); + return; + } + + isShuttingDown = true; + logger.info(`Received ${signal}. Starting graceful shutdown...`); + + try { + // 1. Stop all consumers + logger.info('Stopping consumers...'); + await stopAllConsumers(); + + // 2. Shutdown plugins + logger.info('Shutting down plugins...'); + await PluginRegistry.shutdownAll(); + + // 3. Disconnect Kafka producers + logger.info('Disconnecting Kafka producers...'); + await disconnectStatusProducer(); + await disconnectDelayedProducer(); + + // 4. Disconnect Redis + logger.info('Disconnecting Redis...'); + await disconnectRedis(); + + // 5. Disconnect MongoDB + if (dbConnection) { + logger.info('Disconnecting MongoDB...'); + await dbConnection.disconnect(); } - isShuttingDown = true; - logger.info(`Received ${signal}. Starting graceful shutdown...`); - - try { - // 1. Stop all consumers - logger.info('Stopping consumers...'); - await stopAllConsumers(); - - // 2. Shutdown plugins - logger.info('Shutting down plugins...'); - await PluginRegistry.shutdownAll(); - - // 3. Disconnect Kafka producers - logger.info('Disconnecting Kafka producers...'); - await disconnectStatusProducer(); - await disconnectDelayedProducer(); - - // 4. Disconnect Redis - logger.info('Disconnecting Redis...'); - await disconnectRedis(); - - logger.success('Graceful shutdown complete'); - process.exit(0); - } catch (err) { - logger.error('Error during graceful shutdown:', err); - process.exit(1); - } + logger.success('Graceful shutdown complete'); + process.exit(0); + } catch (err) { + logger.error('Error during graceful shutdown:', err); + process.exit(1); + } }; /** * Register shutdown handlers */ const registerShutdownHandlers = (): void => { - process.on('SIGTERM', () => gracefulShutdown('SIGTERM')); - process.on('SIGINT', () => gracefulShutdown('SIGINT')); - - process.on('uncaughtException', async (err) => { - logger.error('Uncaught exception:', err); - - void AdminAlertService.sendAlert('service_health', - `🔴 UNCAUGHT EXCEPTION - Unified Processor\n` + - `Channels: ${activeChannels.join(', ')}\n` + - `Error: ${err instanceof Error ? err.message : String(err)}\n` + - `Stack: ${err instanceof Error ? err.stack?.slice(0, 200) : 'N/A'}\n` + - `Action: Check processor logs. Process is shutting down.`, - { severity: 'critical' }); - - await gracefulShutdown('uncaughtException'); - }); - - process.on('unhandledRejection', async (reason) => { - logger.error('Unhandled rejection:', reason); - - void AdminAlertService.sendAlert('service_health', - `🔴 UNHANDLED REJECTION - Unified Processor\n` + - `Channels: ${activeChannels.join(', ')}\n` + - `Reason: ${reason instanceof Error ? reason.message : String(reason)}\n` + - `Action: Check processor logs. Process is shutting down.`, - { severity: 'critical' }); - - await gracefulShutdown('unhandledRejection'); - }); + process.on('SIGTERM', () => gracefulShutdown('SIGTERM')); + process.on('SIGINT', () => gracefulShutdown('SIGINT')); + + process.on('uncaughtException', async (err) => { + logger.error('Uncaught exception:', err); + + void AdminAlertService.sendAlert( + 'service_health', + `🔴 UNCAUGHT EXCEPTION - Unified Processor\n` + + `Channels: ${activeChannels.join(', ')}\n` + + `Error: ${err instanceof Error ? err.message : String(err)}\n` + + `Stack: ${err instanceof Error ? err.stack?.slice(0, 200) : 'N/A'}\n` + + `Action: Check processor logs. Process is shutting down.`, + { severity: 'critical' } + ); + + await gracefulShutdown('uncaughtException'); + }); + + process.on('unhandledRejection', async (reason) => { + logger.error('Unhandled rejection:', reason); + + void AdminAlertService.sendAlert( + 'service_health', + `🔴 UNHANDLED REJECTION - Unified Processor\n` + + `Channels: ${activeChannels.join(', ')}\n` + + `Reason: ${reason instanceof Error ? reason.message : String(reason)}\n` + + `Action: Check processor logs. Process is shutting down.`, + { severity: 'critical' } + ); + + await gracefulShutdown('unhandledRejection'); + }); }; /** * Main entry point */ const main = async (): Promise => { - logger.info('================================'); - logger.info('Starting Unified Processor...'); - - try { - // 1. Connect to Redis - logger.info('Connecting to Redis...'); - await connectRedis(); - - // 2. Load plugins from configuration - logger.info('Loading plugins from configuration...'); - await loadProvidersFromEnv(); - - const loadedChannels = PluginRegistry.getChannels(); - if (loadedChannels.length === 0) { - logger.warn('No plugins loaded! Check simplens.config.yaml'); - } else { - logger.info(`Loaded plugins for channels: ${loadedChannels.join(', ')}`); - } + logger.info('================================'); + logger.info('Starting Unified Processor...'); + + try { + // 1. Connect to MongoDB + logger.info('Connecting to MongoDB...'); + dbConnection = await connectMongoDB(); + logger.success('Successfully connected to MongoDB'); + + // 2. Connect to Redis + logger.info('Connecting to Redis...'); + await connectRedis(); + + // 3. Initialize Redis Pub/Sub synchronization + await PluginSyncService.startSubscriber(); + PluginManagerService.registerSyncHandlers(); + ProviderManagerService.registerSyncHandlers(); + ChannelRoutingService.registerSyncHandlers(); + + // Sync consumers when reload or channel route changes occur + PluginSyncService.on('SYSTEM_RELOAD', async () => { + logger.info('Received SYSTEM_RELOAD. Reloading providers from MongoDB...'); + await loadProvidersFromDatabase({ initialize: true }); + await syncActiveConsumers(); + }); - // 3. Initialize Kafka producers - logger.info('Initializing Kafka producers...'); - await initStatusProducer(); - await initDelayedProducer(); - - // 4. Determine which channels to process - const channelConfig = getChannelConfig(); - let channelsToProcess: string[]; - - if (channelConfig === 'all') { - channelsToProcess = loadedChannels; - logger.info('Running in multi-channel mode (all channels)'); - } else { - // Validate requested channels - channelsToProcess = channelConfig.filter(ch => { - if (!loadedChannels.includes(ch)) { - logger.warn(`Channel '${ch}' not found in loaded plugins, skipping`); - return false; - } - return true; - }); - logger.info(`Running in single-channel mode: ${channelsToProcess.join(', ')}`); - } + PluginSyncService.on('CHANNEL_ROUTING_UPDATED', async () => { + await syncActiveConsumers(); + }); - if (channelsToProcess.length === 0) { - logger.error('No valid channels to process! Exiting...'); + PluginSyncService.on('PROVIDER_UPSERTED', async () => { + await syncActiveConsumers(); + }); - void AdminAlertService.sendAlert('service_health', - `🔴 STARTUP FAILURE - Unified Processor\n` + - `Error: No valid channels to process\n` + - `Action: Check simplens.config.yaml. Verify plugins are installed.`, - { severity: 'critical' }); + // 4. Load plugins and providers from MongoDB + logger.info('Loading providers from MongoDB...'); + await loadProvidersFromDatabase({ initialize: true }); - process.exit(1); - } + const loadedChannels = PluginRegistry.getChannels(); + if (loadedChannels.length === 0) { + logger.warn('No providers configured in MongoDB yet. Processor entering idle standby mode...'); + } else { + logger.info(`Loaded plugins for channels: ${loadedChannels.join(', ')}`); + } - // 5. Start consumers for each channel - for (const channel of channelsToProcess) { - logger.info(`Starting consumer for channel: ${channel}`); - await startUnifiedConsumer(channel); - activeChannels.push(channel); + // 5. Initialize Kafka producers + logger.info('Initializing Kafka producers...'); + await initStatusProducer(); + await initDelayedProducer(); + + // 6. Determine which channels to process + const channelConfig = getChannelConfig(); + let channelsToProcess: string[] = []; + + if (channelConfig === 'all') { + channelsToProcess = loadedChannels; + logger.info('Configured for multi-channel mode (all channels)'); + } else { + channelsToProcess = channelConfig.filter((ch) => { + if (!loadedChannels.includes(ch)) { + logger.warn(`Channel '${ch}' not found in loaded plugins, skipping`); + return false; } + return true; + }); + logger.info(`Configured for single-channel mode: ${channelsToProcess.join(', ')}`); + } - // 6. Register shutdown handlers - registerShutdownHandlers(); - - logger.info('================================'); - logger.success(`Unified Processor is running! Channels: ${activeChannels.join(', ')}`); - - } catch (err) { - logger.error('Failed to start Unified Processor:', err); + // 7. Start consumers for each channel (or wait in standby mode) + if (channelsToProcess.length === 0) { + logger.info('Unified Processor is in idle standby mode waiting for channel and provider configuration.'); + } else { + for (const channel of channelsToProcess) { + logger.info(`Starting consumer for channel: ${channel}`); + await startUnifiedConsumer(channel); + activeChannels.push(channel); + } + logger.success(`Unified Processor is running! Active channels: ${activeChannels.join(', ')}`); + } - void AdminAlertService.sendAlert('service_health', - `🔴 STARTUP FAILURE - Unified Processor\n` + - `Error: ${err instanceof Error ? err.message : String(err)}\n` + - `Action: Check processor logs. Verify Redis and Kafka connectivity.`, - { severity: 'critical' }); + // 8. Register shutdown handlers + registerShutdownHandlers(); - process.exit(1); - } + logger.info('================================'); + } catch (err) { + logger.error('Failed to start Unified Processor:', err); + + void AdminAlertService.sendAlert( + 'service_health', + `🔴 STARTUP FAILURE - Unified Processor\n` + + `Error: ${err instanceof Error ? err.message : String(err)}\n` + + `Action: Check processor logs. Verify Redis, MongoDB, and Kafka connectivity.`, + { severity: 'critical' } + ); + + process.exit(1); + } }; // Start the processor diff --git a/src/types/schemas.ts b/src/types/schemas.ts index 099ca7b..af447cc 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -243,7 +243,72 @@ export const adminChannelSchema = z.object({ */ export const systemConfigSchema = z.object({ key: z.string(), - value: z.string(), + value: z.unknown(), + created_at: z.coerce.date().optional(), + updated_at: z.coerce.date().optional(), +}); + +/** + * Key-pair envelope encrypted credentials schema + */ +export const encryptedCredentialsSchema = z.object({ + encrypted_data: z.string(), + iv: z.string(), + auth_tag: z.string(), + encrypted_dek: z.string(), +}); + +/** + * Provider manifest schema for plugins + */ +export const providerManifestSchema = z.object({ + name: z.string(), + version: z.string(), + channel: z.string(), + displayName: z.string(), + description: z.string(), + author: z.string().optional(), + homepage: z.string().optional(), + requiredCredentials: z.array(z.string()), + optionalConfig: z.array(z.string()).optional(), +}); + +/** + * Plugin document schema + */ +export const pluginSchema = z.object({ + name: z.string().min(1), + version: z.string().min(1), + status: z.enum(['installed', 'installing', 'failed']), + error: z.string().optional(), + manifest: providerManifestSchema, + created_at: z.coerce.date().optional(), + updated_at: z.coerce.date().optional(), +}); + +/** + * Provider document schema + */ +export const providerSchema = z.object({ + id: z.string().min(1), + plugin_name: z.string().min(1), + channel: z.string().min(1), + priority: z.number().int().default(0), + enabled: z.boolean().default(true), + credentials: encryptedCredentialsSchema, + options: z.record(z.string(), z.unknown()).optional(), + created_at: z.coerce.date().optional(), + updated_at: z.coerce.date().optional(), +}); + +/** + * Channel routing document schema + */ +export const channelRoutingSchema = z.object({ + channel: z.string().min(1), + default_provider_id: z.string().min(1), + fallback_provider_ids: z.array(z.string()).default([]), + partitions: z.number().int().min(1).default(6), created_at: z.coerce.date().optional(), updated_at: z.coerce.date().optional(), }); diff --git a/src/types/types.ts b/src/types/types.ts index a3de578..8a36f75 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -87,6 +87,12 @@ import { ADMIN_CHANNEL_TYPE, ADMIN_ALERT_TYPE, notificationTemplateSchema, + // Plugin & Provider schemas + encryptedCredentialsSchema, + providerManifestSchema, + pluginSchema, + providerSchema, + channelRoutingSchema, } from "./schemas.js"; // ============================================================================ @@ -124,3 +130,13 @@ export type telegram_config = z.infer; export type AdminChannelType = (typeof ADMIN_CHANNEL_TYPE)[number]; export type AdminAlertType = (typeof ADMIN_ALERT_TYPE)[number]; export type notification_template = z.infer; + +// ============================================================================ +// PLUGIN & PROVIDER MANAGEMENT TYPES +// ============================================================================ + +export type encrypted_credentials = z.infer; +export type provider_manifest = z.infer; +export type plugin_document = z.infer; +export type provider_document = z.infer; +export type channel_routing_document = z.infer; diff --git a/tests/unit/plugins/channel-routing.service.test.ts b/tests/unit/plugins/channel-routing.service.test.ts new file mode 100644 index 0000000..bcf4f99 --- /dev/null +++ b/tests/unit/plugins/channel-routing.service.test.ts @@ -0,0 +1,114 @@ +/** + * Unit Tests for ChannelRoutingService + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { ChannelRoutingService } from '../../../src/plugins/services/channel-routing.service.js'; +import ChannelRouting from '../../../src/database/models/channel-routing.models.js'; +import Provider from '../../../src/database/models/provider.models.js'; +import { PluginRegistry } from '../../../src/plugins/loader/registry.js'; +import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; +import * as kafkaConfig from '../../../src/config/kafka.config.js'; + +describe('ChannelRoutingService', () => { + beforeEach(() => { + vi.restoreAllMocks(); + PluginRegistry.clear(); + vi.spyOn(PluginSyncService, 'publish').mockResolvedValue(undefined); + }); + + describe('setChannelRouting', () => { + it('should throw error if default provider is not found', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue(null); + + await expect( + ChannelRoutingService.setChannelRouting('email', { + default_provider_id: 'nonexistent', + }) + ).rejects.toThrow("Default provider 'nonexistent' not found."); + }); + + it('should throw error if default provider belongs to a different channel', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ + id: 'sms-tw', + channel: 'sms', + } as any); + + await expect( + ChannelRoutingService.setChannelRouting('email', { + default_provider_id: 'sms-tw', + }) + ).rejects.toThrow( + "Default provider 'sms-tw' is configured for channel 'sms', not 'email'." + ); + }); + + it('should throw error if partitions are decreased', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ + id: 'email-primary', + channel: 'email', + } as any); + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue({ + channel: 'email', + partitions: 8, + } as any); + + await expect( + ChannelRoutingService.setChannelRouting('email', { + default_provider_id: 'email-primary', + partitions: 4, + }) + ).rejects.toThrow( + "Cannot decrease partitions for channel 'email' from 8 to 4. Kafka partitions can only be increased." + ); + }); + + it('should expand Kafka partitions when partition count is increased and update DB', async () => { + vi.spyOn(Provider, 'findOne').mockImplementation(async (query: any) => { + if (query.id === 'email-primary') { + return { id: 'email-primary', channel: 'email' } as any; + } + if (query.id === 'email-fb-1') { + return { id: 'email-fb-1', channel: 'email' } as any; + } + return null; + }); + + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue({ + channel: 'email', + partitions: 6, + } as any); + + const expandSpy = vi + .spyOn(kafkaConfig, 'expandTopicPartitions') + .mockResolvedValue(undefined); + + const updatedRouting = { + channel: 'email', + default_provider_id: 'email-primary', + fallback_provider_ids: ['email-fb-1'], + partitions: 12, + toObject: () => updatedRouting, + }; + + vi.spyOn(ChannelRouting, 'findOneAndUpdate').mockResolvedValue(updatedRouting as any); + const setConfigSpy = vi.spyOn(PluginRegistry, 'setChannelConfig'); + + const result = await ChannelRoutingService.setChannelRouting('email', { + default_provider_id: 'email-primary', + fallback_provider_ids: ['email-fb-1'], + partitions: 12, + }); + + expect(expandSpy).toHaveBeenCalledWith('email', 12); + expect(setConfigSpy).toHaveBeenCalledWith('email', { + default: 'email-primary', + fallback: ['email-fb-1'], + }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('CHANNEL_ROUTING_UPDATED', { + channel: 'email', + }); + expect(result.partitions).toBe(12); + }); + }); +}); diff --git a/tests/unit/plugins/crypto.test.ts b/tests/unit/plugins/crypto.test.ts new file mode 100644 index 0000000..419ced0 --- /dev/null +++ b/tests/unit/plugins/crypto.test.ts @@ -0,0 +1,118 @@ +/** + * Unit Tests for Key-Pair Envelope Encryption Manager + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { + generateRSAKeyPair, + encryptCredentials, + decryptCredentials, + clearKeyPairCache, +} from '../../../src/plugins/crypto/keypair-manager.js'; +import system_config_model from '../../../src/database/models/system-config.models.js'; + +describe('Key-Pair Envelope Encryption Manager', () => { + beforeEach(() => { + clearKeyPairCache(); + vi.restoreAllMocks(); + }); + + describe('generateRSAKeyPair', () => { + it('should generate valid RSA 2048-bit public and private PEM keys', () => { + const keyPair = generateRSAKeyPair(); + expect(keyPair.publicKey).toContain('-----BEGIN PUBLIC KEY-----'); + expect(keyPair.publicKey).toContain('-----END PUBLIC KEY-----'); + expect(keyPair.privateKey).toContain('-----BEGIN PRIVATE KEY-----'); + expect(keyPair.privateKey).toContain('-----END PRIVATE KEY-----'); + }); + }); + + describe('encryptCredentials & decryptCredentials', () => { + it('should encrypt and decrypt credentials successfully with mock database', async () => { + const mockKeyPair = generateRSAKeyPair(); + + // Mock MongoDB system_config_model + vi.spyOn(system_config_model, 'findOne').mockImplementation(async (query: any) => { + if (query.key === 'plugin_credentials_public_key') { + return { key: 'plugin_credentials_public_key', value: mockKeyPair.publicKey } as any; + } + if (query.key === 'plugin_credentials_private_key') { + return { key: 'plugin_credentials_private_key', value: mockKeyPair.privateKey } as any; + } + return null; + }); + + const credentials = { + GMAIL_USER: 'test@gmail.com', + GMAIL_APP_PASSWORD: 'app-password-secret-123', + API_TOKEN: 'token_xyz_987654321', + }; + + const encrypted = await encryptCredentials(credentials); + + expect(encrypted).toHaveProperty('encrypted_data'); + expect(encrypted).toHaveProperty('iv'); + expect(encrypted).toHaveProperty('auth_tag'); + expect(encrypted).toHaveProperty('encrypted_dek'); + + expect(typeof encrypted.encrypted_data).toBe('string'); + expect(typeof encrypted.iv).toBe('string'); + expect(typeof encrypted.auth_tag).toBe('string'); + expect(typeof encrypted.encrypted_dek).toBe('string'); + + // Now decrypt + const decrypted = await decryptCredentials(encrypted); + expect(decrypted).toEqual(credentials); + }); + + it('should generate new keys and save them if not found in database or env', async () => { + const storedConfigs = new Map(); + + vi.spyOn(system_config_model, 'findOne').mockImplementation(async (query: any) => { + const val = storedConfigs.get(query.key); + return val ? ({ key: query.key, value: val } as any) : null; + }); + + vi.spyOn(system_config_model, 'findOneAndUpdate').mockImplementation(async (query: any, update: any) => { + storedConfigs.set(query.key, update.value); + return { key: query.key, value: update.value } as any; + }); + + const creds = { API_KEY: 'super_secret' }; + const encrypted = await encryptCredentials(creds); + + expect(storedConfigs.has('plugin_credentials_public_key')).toBe(true); + expect(storedConfigs.has('plugin_credentials_private_key')).toBe(true); + + const decrypted = await decryptCredentials(encrypted); + expect(decrypted).toEqual(creds); + }); + + it('should handle large credential payloads through hybrid AES envelope', async () => { + const mockKeyPair = generateRSAKeyPair(); + + vi.spyOn(system_config_model, 'findOne').mockImplementation(async (query: any) => { + if (query.key === 'plugin_credentials_public_key') { + return { key: 'plugin_credentials_public_key', value: mockKeyPair.publicKey } as any; + } + if (query.key === 'plugin_credentials_private_key') { + return { key: 'plugin_credentials_private_key', value: mockKeyPair.privateKey } as any; + } + return null; + }); + + // 10KB payload (exceeds direct RSA-2048 OAEP limit of ~214 bytes) + const largeCredentials = { + SERVICE_ACCOUNT_JSON: JSON.stringify({ + type: 'service_account', + private_key: '-----BEGIN PRIVATE KEY-----\n' + 'MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC...'.repeat(150) + '\n-----END PRIVATE KEY-----', + }), + }; + + const encrypted = await encryptCredentials(largeCredentials); + const decrypted = await decryptCredentials(encrypted); + + expect(decrypted).toEqual(largeCredentials); + }); + }); +}); diff --git a/tests/unit/plugins/plugin-manager.service.test.ts b/tests/unit/plugins/plugin-manager.service.test.ts new file mode 100644 index 0000000..1f3750a --- /dev/null +++ b/tests/unit/plugins/plugin-manager.service.test.ts @@ -0,0 +1,142 @@ +/** + * Unit Tests for PluginManagerService + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { PluginManagerService } from '../../../src/plugins/services/plugin-manager.service.js'; +import Plugin from '../../../src/database/models/plugin.models.js'; +import Provider from '../../../src/database/models/provider.models.js'; +import * as pluginFs from '../../../src/plugins/loader/plugin-fs.js'; +import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; + +describe('PluginManagerService', () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.spyOn(PluginSyncService, 'publish').mockResolvedValue(undefined); + }); + + describe('installPlugin', () => { + it('should install npm package, extract manifest with package.json version, save to DB, and publish event', async () => { + const installSpy = vi.spyOn(pluginFs, 'installNpmPackage').mockReturnValue(undefined); + vi.spyOn(pluginFs, 'extractPackageManifest').mockResolvedValue({ + manifest: { + name: '@simplens/mock', + displayName: 'Mock Provider', + version: '2.1.0', + channel: 'mock', + description: 'Mock plugin', + requiredCredentials: [], + }, + version: '2.1.0', + }); + + const mockPluginDoc = { + name: '@simplens/mock', + version: '2.1.0', + status: 'installed', + manifest: { + name: '@simplens/mock', + displayName: 'Mock Provider', + version: '2.1.0', + channel: 'mock', + description: 'Mock plugin', + requiredCredentials: [], + }, + toObject: () => ({ + name: '@simplens/mock', + version: '2.1.0', + status: 'installed', + }), + }; + + vi.spyOn(Plugin, 'findOneAndUpdate').mockResolvedValue(mockPluginDoc as any); + + const result = await PluginManagerService.installPlugin('@simplens/mock', '2.1.0'); + + expect(installSpy).toHaveBeenCalledWith('@simplens/mock', '2.1.0'); + expect(Plugin.findOneAndUpdate).toHaveBeenCalledWith( + { name: '@simplens/mock' }, + expect.objectContaining({ + name: '@simplens/mock', + version: '2.1.0', + status: 'installed', + }), + { upsert: true, new: true } + ); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PLUGIN_INSTALLED', { + plugin_name: '@simplens/mock', + version: '2.1.0', + }); + expect(result.version).toBe('2.1.0'); + }); + }); + + describe('changePluginVersion', () => { + it('should throw error if plugin is not already installed', async () => { + vi.spyOn(Plugin, 'findOne').mockResolvedValue(null); + + await expect( + PluginManagerService.changePluginVersion('@simplens/unknown', '1.2.0') + ).rejects.toThrow("Plugin '@simplens/unknown' is not installed."); + }); + + it('should install new version and update DB when plugin exists', async () => { + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ name: '@simplens/mock' } as any); + vi.spyOn(pluginFs, 'installNpmPackage').mockReturnValue(undefined); + vi.spyOn(pluginFs, 'extractPackageManifest').mockResolvedValue({ + manifest: { + name: '@simplens/mock', + displayName: 'Mock Provider', + version: '3.0.0', + channel: 'mock', + description: 'Mock plugin v3', + requiredCredentials: [], + }, + version: '3.0.0', + }); + + const updatedDoc = { + name: '@simplens/mock', + version: '3.0.0', + toObject: () => ({ + name: '@simplens/mock', + version: '3.0.0', + }), + }; + vi.spyOn(Plugin, 'findOneAndUpdate').mockResolvedValue(updatedDoc as any); + + const result = await PluginManagerService.changePluginVersion('@simplens/mock', '3.0.0'); + + expect(PluginSyncService.publish).toHaveBeenCalledWith('PLUGIN_VERSION_CHANGED', { + plugin_name: '@simplens/mock', + version: '3.0.0', + }); + expect(result.version).toBe('3.0.0'); + }); + }); + + describe('uninstallPlugin', () => { + it('should reject uninstallation if providers depend on the plugin', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(2 as any); + + await expect(PluginManagerService.uninstallPlugin('@simplens/mock')).rejects.toThrow( + "Cannot uninstall plugin '@simplens/mock': 2 provider(s) depend on it." + ); + }); + + it('should uninstall and remove from DB when no providers depend on it', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ name: '@simplens/mock' } as any); + const uninstallSpy = vi.spyOn(pluginFs, 'uninstallNpmPackage').mockReturnValue(undefined); + const deleteSpy = vi.spyOn(Plugin, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); + + await PluginManagerService.uninstallPlugin('@simplens/mock'); + + expect(uninstallSpy).toHaveBeenCalledWith('@simplens/mock'); + expect(deleteSpy).toHaveBeenCalledWith({ name: '@simplens/mock' }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PLUGIN_UNINSTALLED', { + plugin_name: '@simplens/mock', + }); + }); + }); +}); diff --git a/tests/unit/plugins/provider-manager.service.test.ts b/tests/unit/plugins/provider-manager.service.test.ts new file mode 100644 index 0000000..5dcdcbd --- /dev/null +++ b/tests/unit/plugins/provider-manager.service.test.ts @@ -0,0 +1,147 @@ +/** + * Unit Tests for ProviderManagerService + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { ProviderManagerService } from '../../../src/plugins/services/provider-manager.service.js'; +import Provider from '../../../src/database/models/provider.models.js'; +import Plugin from '../../../src/database/models/plugin.models.js'; +import ChannelRouting from '../../../src/database/models/channel-routing.models.js'; +import * as keypairManager from '../../../src/plugins/crypto/keypair-manager.js'; +import * as pluginFs from '../../../src/plugins/loader/plugin-fs.js'; +import { PluginRegistry } from '../../../src/plugins/loader/registry.js'; +import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; + +describe('ProviderManagerService', () => { + beforeEach(() => { + vi.restoreAllMocks(); + PluginRegistry.clear(); + vi.spyOn(PluginSyncService, 'publish').mockResolvedValue(undefined); + }); + + describe('createProvider', () => { + it('should throw error if provider ID already exists', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ id: 'gmail-prod' } as any); + + await expect( + ProviderManagerService.createProvider({ + id: 'gmail-prod', + plugin_name: '@simplens/nodemailer-gmail', + credentials: { EMAIL_USER: 'test@example.com' }, + }) + ).rejects.toThrow("Provider with ID 'gmail-prod' already exists."); + }); + + it('should throw error if plugin is not installed', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue(null); + vi.spyOn(Plugin, 'findOne').mockResolvedValue(null); + + await expect( + ProviderManagerService.createProvider({ + id: 'gmail-prod', + plugin_name: '@simplens/nodemailer-gmail', + credentials: { EMAIL_USER: 'test@example.com' }, + }) + ).rejects.toThrow("Plugin '@simplens/nodemailer-gmail' is not installed."); + }); + + it('should encrypt credentials, create provider in DB, and broadcast sync event', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue(null); + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ + name: '@simplens/mock', + manifest: { channel: 'mock' }, + } as any); + + vi.spyOn(keypairManager, 'encryptCredentials').mockResolvedValue({ + encrypted_data: 'enc123', + iv: 'iv123', + auth_tag: 'tag123', + encrypted_dek: 'dek123', + }); + + const mockCreated = { + _id: '507f1f77bcf86cd799439011', + id: 'mock-test', + plugin_name: '@simplens/mock', + channel: 'mock', + priority: 5, + enabled: true, + credentials: { encrypted_data: 'enc123' }, + options: {}, + toObject: () => mockCreated, + }; + + vi.spyOn(Provider, 'create').mockResolvedValue(mockCreated as any); + vi.spyOn(ProviderManagerService, 'loadAndRegisterProvider').mockResolvedValue(undefined); + + const result = await ProviderManagerService.createProvider({ + id: 'mock-test', + plugin_name: '@simplens/mock', + credentials: {}, + priority: 5, + }); + + expect(keypairManager.encryptCredentials).toHaveBeenCalled(); + expect(Provider.create).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'mock-test', + plugin_name: '@simplens/mock', + channel: 'mock', + priority: 5, + }) + ); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_UPSERTED', { + provider_id: 'mock-test', + }); + expect(result.credentials_configured).toBe(true); + }); + }); + + describe('deleteProvider', () => { + it('should reject deletion if provider is used in channel routing', async () => { + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue({ + channel: 'email', + default_provider_id: 'gmail-prod', + } as any); + + await expect(ProviderManagerService.deleteProvider('gmail-prod')).rejects.toThrow( + "Cannot delete provider 'gmail-prod': it is configured in channel 'email' routing." + ); + }); + + it('should delete from DB and unregister from registry if not in use', async () => { + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue(null); + vi.spyOn(Provider, 'findOne').mockResolvedValue({ id: 'mock-unused' } as any); + const deleteSpy = vi.spyOn(Provider, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); + const unregisterSpy = vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); + + await ProviderManagerService.deleteProvider('mock-unused'); + + expect(deleteSpy).toHaveBeenCalledWith({ id: 'mock-unused' }); + expect(unregisterSpy).toHaveBeenCalledWith('mock-unused'); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_DELETED', { + provider_id: 'mock-unused', + }); + }); + }); + + describe('testProviderConnection', () => { + it('should return success true when health check passes', async () => { + const mockProviderInstance = { + initialize: vi.fn().mockResolvedValue(undefined), + healthCheck: vi.fn().mockResolvedValue(true), + }; + + vi.spyOn(pluginFs, 'importAndInstantiateProvider').mockResolvedValue(mockProviderInstance as any); + + const result = await ProviderManagerService.testProviderConnection({ + plugin_name: '@simplens/mock', + credentials: {}, + }); + + expect(result.success).toBe(true); + expect(mockProviderInstance.initialize).toHaveBeenCalled(); + expect(mockProviderInstance.healthCheck).toHaveBeenCalled(); + }); + }); +}); diff --git a/tests/unit/plugins/router.test.ts b/tests/unit/plugins/router.test.ts index 7fdd848..6340810 100644 --- a/tests/unit/plugins/router.test.ts +++ b/tests/unit/plugins/router.test.ts @@ -22,6 +22,7 @@ vi.mock('../../../src/processors/unified/unified.logger.js', () => ({ warn: vi.fn(), error: vi.fn(), debug: vi.fn(), + success: vi.fn(), }, })); diff --git a/tsconfig.json b/tsconfig.json index 83afc3e..1d5fb31 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,7 @@ { "compilerOptions": { "target": "ES2020", + "lib": ["ES2022"], "module": "NodeNext", "moduleResolution": "NodeNext", "baseUrl": ".", From beba7895a7fb4f1a1ab214b674719d38287792d6 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Tue, 15 Sep 2026 00:42:03 +0530 Subject: [PATCH 04/46] feat: refactor types and interfaces for improved organization and clarity across the codebase --- src/admin-alerts/admin-channel.interface.ts | 75 +---- src/admin-alerts/channel-registry.ts | 9 +- src/plugins/bootstrap/yaml-migrator.ts | 25 +- src/plugins/interfaces/provider.types.ts | 99 +----- src/plugins/loader/loader.ts | 35 +- src/plugins/loader/registry.ts | 64 +--- .../services/provider-manager.service.ts | 16 +- src/plugins/sync/plugin-sync.service.ts | 28 +- src/processors/delayed/delayed.consumer.ts | 11 +- src/processors/delayed/delayed.poller.ts | 6 +- src/processors/shared/idempotency.ts | 7 +- src/processors/shared/rate-limiter.ts | 17 +- src/types/schemas.ts | 89 +++-- src/types/types.ts | 316 ++++++++++++++++++ src/workers/consumers/status.consumer.ts | 22 +- src/workers/cron/background.cron.ts | 14 +- src/workers/recovery/recovery.cron.ts | 17 +- src/workers/utils/logger.ts | 26 +- src/workers/utils/types.ts | 15 +- src/workers/utils/validation.ts | 12 +- 20 files changed, 409 insertions(+), 494 deletions(-) diff --git a/src/admin-alerts/admin-channel.interface.ts b/src/admin-alerts/admin-channel.interface.ts index 737d54f..99003c7 100644 --- a/src/admin-alerts/admin-channel.interface.ts +++ b/src/admin-alerts/admin-channel.interface.ts @@ -3,7 +3,12 @@ * Defines the contract for admin notification channel implementations */ -import type { AdminAlertType, AdminChannelType } from '@src/types/types.js'; +export type { + AdminChannelProvider, + AlertMetadata, + ChannelResult, + CredentialField, +} from '@src/types/types.js'; /** * Severity colors for Discord embeds @@ -28,72 +33,4 @@ export const ALERT_EMOJIS: Record = { /** * Result of a channel operation */ -export interface ChannelResult { - success: boolean; - error?: string; -} - -/** - * Metadata attached to alerts for rich formatting - */ -export interface AlertMetadata { - alertType?: AdminAlertType; - severity?: 'info' | 'warning' | 'critical'; - timestamp?: Date; - notificationId?: string; - channel?: string; - errorMessage?: string; -} - -/** - * Credential field definition for dynamic form generation - * Used by dashboard to render provider-specific configuration forms - */ -export interface CredentialField { - /** Field name (used as key in config object) */ - name: string; - /** Field type for input rendering */ - type: 'string' | 'url' | 'secret'; - /** Display label for the field */ - label: string; - /** Placeholder text for input */ - placeholder?: string; - /** Help text / description */ - description?: string; - /** Whether field is required */ - required: boolean; - /** Regex pattern for validation */ - pattern?: string; -} - -/** - * Interface for admin alert channel providers - * All channel implementations must implement this interface - */ -export interface AdminChannelProvider { - /** The channel type identifier */ - readonly channelType: AdminChannelType; - - /** Display name for UI */ - readonly displayName: string; - - /** - * Send an alert message through this channel - * @param message - The alert message to send - * @param metadata - Optional metadata for rich formatting - */ - send(message: string, metadata?: AlertMetadata): Promise; - - /** - * Test the channel connection - * @returns Result indicating if the connection test was successful - */ - testConnection(): Promise; - - /** - * Get credential schema for dynamic form generation - * @returns Array of credential field definitions - */ - getCredentialSchema(): CredentialField[]; -} diff --git a/src/admin-alerts/channel-registry.ts b/src/admin-alerts/channel-registry.ts index f46acd4..eba8ce8 100644 --- a/src/admin-alerts/channel-registry.ts +++ b/src/admin-alerts/channel-registry.ts @@ -3,8 +3,7 @@ * Extensible pattern - new channels register themselves here */ -import type { AdminChannelProvider, CredentialField } from './admin-channel.interface.js'; -import type { AdminChannelType } from '@src/types/types.js'; +import type { AdminChannelProvider, AdminChannelMeta, AdminChannelType } from '@src/types/types.js'; /** * Factory function type for creating channel provider instances @@ -67,12 +66,6 @@ export function getRegisteredChannelTypes(): AdminChannelType[] { /** * Admin channel metadata for dashboard */ -export interface AdminChannelMeta { - channelType: AdminChannelType; - displayName: string; - credentialFields: CredentialField[]; -} - /** * Get metadata for all registered admin channel providers * Used by dashboard for dynamic form generation diff --git a/src/plugins/bootstrap/yaml-migrator.ts b/src/plugins/bootstrap/yaml-migrator.ts index 741121a..4a07cfa 100644 --- a/src/plugins/bootstrap/yaml-migrator.ts +++ b/src/plugins/bootstrap/yaml-migrator.ts @@ -21,30 +21,7 @@ import { } from '@src/plugins/loader/plugin-fs.js'; import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; - -interface YamlProviderEntry { - package: string; - version?: string; - id: string; - credentials?: Record; - optionalConfig?: Record; - options?: { - priority?: number; - [key: string]: unknown; - }; -} - -interface YamlConfig { - providers?: YamlProviderEntry[]; - channels?: Record< - string, - { - default: string; - fallback?: string | string[]; - partitions?: number; - } - >; -} +import type { YamlConfig } from '@src/types/types.js'; /** * Resolve environment variables in credential templates diff --git a/src/plugins/interfaces/provider.types.ts b/src/plugins/interfaces/provider.types.ts index 50c0cfe..bab2ec2 100644 --- a/src/plugins/interfaces/provider.types.ts +++ b/src/plugins/interfaces/provider.types.ts @@ -1,89 +1,12 @@ /** - * Provider Types - * - * Type definitions for SimpleNS provider plugins. - * These mirror the @simplens/sdk interfaces for use within the core. - */ - -import { z } from 'zod'; - -/** - * Delivery result from sending a notification - */ -export interface DeliveryResult { - success: boolean; - messageId?: string; - providerResponse?: unknown; - error?: { - code: string; - message: string; - retryable: boolean; - }; -} - -/** - * Provider configuration passed during initialization - */ -export interface ProviderConfig { - id: string; - credentials: Record; - options?: Record; -} - -/** - * Rate limit configuration - */ -export interface RateLimitConfig { - maxTokens: number; - refillRate: number; - refillInterval?: 'second' | 'minute' | 'hour' | 'day'; -} - -/** - * Provider manifest - metadata about the provider - */ -export interface ProviderManifest { - name: string; - version: string; - channel: string; - displayName: string; - description: string; - author: string; - homepage?: string; - requiredCredentials: string[]; - optionalConfig?: string[]; -} - -/** - * Base notification fields common to all channels - */ -export interface BaseNotification { - notification_id: string; - request_id: string; - client_id: string; - channel: string; - provider?: string; - recipient: Record; - content: Record; - variables?: Record; - webhook_url: string; - retry_count: number; - created_at: Date; -} - -/** - * SimpleNS Provider Interface - */ -export interface SimpleNSProvider { - readonly manifest: ProviderManifest; - - getNotificationSchema(): z.ZodSchema; - getRecipientSchema(): z.ZodObject>; - getContentSchema(): z.ZodObject>; - getRateLimitConfig(): RateLimitConfig; - - initialize(config: ProviderConfig): Promise; - healthCheck(): Promise; - send(notification: TNotification): Promise; - shutdown(): Promise; -} + * Provider contracts are owned by the shared type module. These exports keep + * the existing plugin import path compatible with external providers. + */ +export type { + BaseNotification, + DeliveryResult, + ProviderConfig, + ProviderManifest, + RateLimitConfig, + SimpleNSProvider, +} from '@src/types/types.js'; diff --git a/src/plugins/loader/loader.ts b/src/plugins/loader/loader.ts index 0451e93..57120e9 100644 --- a/src/plugins/loader/loader.ts +++ b/src/plugins/loader/loader.ts @@ -18,7 +18,7 @@ import Provider from '@src/database/models/provider.models.js'; import ChannelRouting from '@src/database/models/channel-routing.models.js'; import { ProviderManagerService } from '@src/plugins/services/provider-manager.service.js'; import { installNpmPackage } from './plugin-fs.js'; -import type { provider_document } from '@src/types/types.js'; +import type { provider_document, ProviderEntry, SimpleNSConfig } from '@src/types/types.js'; // Plugins directory for user-installed plugins const PLUGINS_DIR = join(process.cwd(), '.plugins'); @@ -91,39 +91,6 @@ async function installMissingPlugins(config: SimpleNSConfig): Promise { /** * Provider entry in configuration */ -interface ProviderEntry { - /** npm package name (e.g., '@simplens/gmail') */ - package: string; - /** Unique ID for this provider instance */ - id: string; - /** Provider credentials */ - credentials: Record; - /** Optional config values (resolved from env vars) */ - optionalConfig?: Record; - /** Optional settings */ - options?: { - priority?: number; - rateLimit?: { - maxTokens?: number; - refillRate?: number; - }; - [key: string]: unknown; - }; -} - -/** - * SimpleNS configuration file structure - */ -interface SimpleNSConfig { - /** Provider configurations */ - providers: ProviderEntry[]; - /** Channel to provider mapping */ - channels?: Record; -} - /** * Find local config file if it exists * Checks for simplens.config.yaml/.yml/.json diff --git a/src/plugins/loader/registry.ts b/src/plugins/loader/registry.ts index ff40753..89be81b 100644 --- a/src/plugins/loader/registry.ts +++ b/src/plugins/loader/registry.ts @@ -5,65 +5,21 @@ * Handles provider lookup by ID or channel. */ -import type { SimpleNSProvider, ProviderManifest } from '../interfaces/provider.types.js'; +import type { SimpleNSProvider, ProviderManifest, RegisteredProvider, ChannelConfig, FieldDefinition, ProviderMetadata, ChannelMetadata, PluginMetadata } from '@src/types/types.js'; import { pluginRegistryLogger as logger } from '@src/workers/utils/logger.js'; -/** - * Registered provider with configuration - */ -export interface RegisteredProvider { - provider: SimpleNSProvider; - id: string; - priority: number; -} - -/** - * Channel configuration - */ -export interface ChannelConfig { - default: string; - fallback?: string | string[]; -} - -/** - * Field definition for dynamic form generation - */ -export interface FieldDefinition { - name: string; - type: 'string' | 'email' | 'phone' | 'text' | 'number' | 'boolean'; - required: boolean; - description?: string; -} +export type { + RegisteredProvider, + ChannelConfig, + FieldDefinition, + ProviderMetadata, + ChannelMetadata, + PluginMetadata, +} from '@src/types/types.js'; /** - * Provider metadata for dashboard - */ -export interface ProviderMetadata { - id: string; - name: string; - displayName: string; - description?: string; - priority: number; - recipientFields: FieldDefinition[]; - contentFields: FieldDefinition[]; -} - -/** - * Channel metadata for dashboard - */ -export interface ChannelMetadata { - providers: ProviderMetadata[]; - default?: string; - fallback?: string | string[]; -} - -/** - * Full plugin metadata response + * Registered provider with configuration */ -export interface PluginMetadata { - channels: Record; -} - /** * Extract field definitions from a Zod object schema */ diff --git a/src/plugins/services/provider-manager.service.ts b/src/plugins/services/provider-manager.service.ts index 9bad510..c10aea4 100644 --- a/src/plugins/services/provider-manager.service.ts +++ b/src/plugins/services/provider-manager.service.ts @@ -15,22 +15,10 @@ import { import { importAndInstantiateProvider } from '@src/plugins/loader/plugin-fs.js'; import { PluginRegistry } from '@src/plugins/loader/registry.js'; import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; -import type { provider_document } from '@src/types/types.js'; -import type { ProviderConfig } from '@src/plugins/interfaces/provider.types.js'; +import type { provider_document, ProviderConfig, ProviderResponseDto } from '@src/types/types.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; -export interface ProviderResponseDto { - _id?: string; - id: string; - plugin_name: string; - channel: string; - priority: number; - enabled: boolean; - options?: Record; - credentials_configured: boolean; - created_at?: Date; - updated_at?: Date; -} +export type { ProviderResponseDto } from '@src/types/types.js'; export class ProviderManagerService { /** diff --git a/src/plugins/sync/plugin-sync.service.ts b/src/plugins/sync/plugin-sync.service.ts index 74a60d6..44f76dc 100644 --- a/src/plugins/sync/plugin-sync.service.ts +++ b/src/plugins/sync/plugin-sync.service.ts @@ -9,36 +9,10 @@ import { randomUUID } from 'crypto'; import type { Redis as RedisType } from 'ioredis'; import { getRedisClient } from '@src/config/redis.config.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; +import type { PluginSyncAction, PluginSyncHandler, PluginSyncMessage, PluginSyncPayload } from '@src/types/types.js'; export const PLUGIN_SYNC_CHANNEL = 'simplens:plugin_system:sync'; -export type PluginSyncAction = - | 'PLUGIN_INSTALLED' - | 'PLUGIN_UNINSTALLED' - | 'PLUGIN_VERSION_CHANGED' - | 'PROVIDER_UPSERTED' - | 'PROVIDER_DELETED' - | 'CHANNEL_ROUTING_UPDATED' - | 'SYSTEM_RELOAD'; - -export interface PluginSyncPayload { - plugin_name?: string; - version?: string; - provider_id?: string; - channel?: string; - [key: string]: unknown; -} - -export interface PluginSyncMessage { - event_id: string; - source_instance_id: string; - timestamp: string; - action: PluginSyncAction; - payload: PluginSyncPayload; -} - -export type PluginSyncHandler = (message: PluginSyncMessage) => Promise; - export class PluginSyncServiceClass { private static instance: PluginSyncServiceClass; private instanceId: string; diff --git a/src/processors/delayed/delayed.consumer.ts b/src/processors/delayed/delayed.consumer.ts index f6c42bc..47c6a6e 100644 --- a/src/processors/delayed/delayed.consumer.ts +++ b/src/processors/delayed/delayed.consumer.ts @@ -3,21 +3,16 @@ * Channel-agnostic - routes based on dynamic target_topic */ -import { Consumer, EachMessagePayload } from 'kafkajs'; +import { EachMessagePayload } from 'kafkajs'; import { kafka } from '@src/config/kafka.config.js'; -import { CORE_TOPICS } from '@src/types/types.js'; +import { CORE_TOPICS, type KafkaConsumerState } from '@src/types/types.js'; import { delayedNotificationTopicSchema } from '@src/types/schemas.js'; import { addToDelayedQueue } from './delayed.queue.js'; import { delayedWorkerLogger as logger } from '@src/workers/utils/logger.js'; const CONSUMER_GROUP_ID = 'delayed-worker-group'; -interface ConsumerState { - consumer: Consumer | null; - isConsuming: boolean; -} - -const state: ConsumerState = { +const state: KafkaConsumerState = { consumer: null, isConsuming: false }; diff --git a/src/processors/delayed/delayed.poller.ts b/src/processors/delayed/delayed.poller.ts index 3697426..1908b0a 100644 --- a/src/processors/delayed/delayed.poller.ts +++ b/src/processors/delayed/delayed.poller.ts @@ -13,7 +13,7 @@ import { claimDueEvents, confirmProcessed, reAddToQueue, getDueEventCount, relea import { publishToTarget } from './target.producer.js'; import { publishDLQFailureStatus } from './dlq.status.js'; import { delayedWorkerLogger as logger } from '@src/workers/utils/logger.js'; -import type { delayed_notification_topic } from '@src/types/types.js'; +import type { DelayedEventWithRetries } from '@src/types/types.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; let pollerInterval: NodeJS.Timeout | null = null; @@ -22,10 +22,6 @@ let isPolling = false; /** * Extended event type with poller retry tracking */ -interface DelayedEventWithRetries extends delayed_notification_topic { - _pollerRetries?: number; -} - /** * Calculate exponential backoff delay * @param retryCount Current retry count diff --git a/src/processors/shared/idempotency.ts b/src/processors/shared/idempotency.ts index 3064dfd..9f2ab03 100644 --- a/src/processors/shared/idempotency.ts +++ b/src/processors/shared/idempotency.ts @@ -6,17 +6,12 @@ import { getRedisClient } from '@src/config/redis.config.js'; import { env } from '@src/config/env.config.js'; +import type { IdempotencyRecord } from '@src/types/types.js'; // Redis key prefix for idempotency const IDEMPOTENCY_PREFIX = 'idempotency'; // Idempotency record stored in Redis -interface IdempotencyRecord { - status: 'processing' | 'delivered' | 'failed' | 'rate_limited'; - retry_count: number; - updated_at: string; -} - /** * Build Redis key for idempotency check */ diff --git a/src/processors/shared/rate-limiter.ts b/src/processors/shared/rate-limiter.ts index 05e2fbb..dadb1e0 100644 --- a/src/processors/shared/rate-limiter.ts +++ b/src/processors/shared/rate-limiter.ts @@ -8,21 +8,13 @@ import { getRedisClient } from '@src/config/redis.config.js'; import { getRateLimitConfig as getPluginRateLimitConfig } from '@src/plugins/index.js'; import { rateLimiterLogger as logger } from '@src/workers/utils/logger.js'; +import type { RateLimitConfig, RefillInterval, RateLimitResult } from '@src/types/types.js'; // Redis key prefixes const TOKENS_KEY_PREFIX = 'ratelimit:tokens'; const LAST_REFILL_KEY_PREFIX = 'ratelimit:last_refill'; const QUEUE_POSITION_KEY_PREFIX = 'ratelimit:queue_position'; -/** Time interval for refill rate */ -type RefillInterval = 'second' | 'minute' | 'hour' | 'day'; - -interface RateLimitConfig { - maxTokens: number; - refillRate: number; - refillInterval?: RefillInterval; -} - // Conversion divisors to convert rate to per-second const INTERVAL_TO_SECONDS: Record = { second: 1, @@ -77,13 +69,6 @@ const buildKeys = (providerId: string): { tokensKey: string; lastRefillKey: stri /** * Token Bucket Algorithm Result */ -export interface RateLimitResult { - allowed: boolean; - remainingTokens: number; - retryAfterMs?: number; - queuePosition?: number; // Position in queue when rate-limited -} - /** * Try to consume a token from the bucket * Uses Redis Lua script for atomic operation with queue-based staggering diff --git a/src/types/schemas.ts b/src/types/schemas.ts index af447cc..c103fa1 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -41,15 +41,7 @@ const hasNonEmptyContent = ( content?: Record>, ) => !!content && Object.keys(content).length > 0; -// ============================================================================ -// BASE NOTIFICATION SCHEMA (Channel-Agnostic) -// ============================================================================ - -/** - * Base notification schema - plugins extend this with channel-specific fields - */ -export const baseNotificationSchema = z.object({ - notification_id: objectIdSchema, +const notificationPayloadFields = { request_id: UUIDV4Schema, client_id: UUIDV4Schema, client_name: z.string().optional(), @@ -59,8 +51,40 @@ export const baseNotificationSchema = z.object({ content: z.record(z.string(), z.unknown()), variables: variablesSchema.optional(), webhook_url: z.url(), - retry_count: z.number().int().min(0), scheduled_at: z.coerce.date().optional(), +}; + +const notificationRequestFields = { + client_id: UUIDV4Schema, + client_name: z.string().optional(), + channel: z.array(z.string()).min(1, "At least one channel is required."), + template_id: z.array(z.string().nullable().optional()).optional(), + provider: z + .union([z.string(), z.array(z.string().nullable().optional())]) + .optional(), + content: z.record(z.string(), z.record(z.string(), z.string())).optional(), + scheduled_at: z.coerce.date().optional(), + webhook_url: z.url(), +}; + +const notificationTemplateFields = { + name: z.string(), + description: z.string().optional(), + content: z.record(z.string(), z.unknown()), + package: z.string(), +}; + +// ============================================================================ +// BASE NOTIFICATION SCHEMA (Channel-Agnostic) +// ============================================================================ + +/** + * Base notification schema - plugins extend this with channel-specific fields + */ +export const baseNotificationSchema = z.object({ + notification_id: objectIdSchema, + ...notificationPayloadFields, + retry_count: z.number().int().min(0), created_at: z.coerce.date(), }); @@ -104,17 +128,8 @@ export const notificationStatusTopicSchema = z.object({ * Notification record in MongoDB */ export const notificationSchema = z.object({ - request_id: UUIDV4Schema, - client_id: UUIDV4Schema, - client_name: z.string().optional(), - channel: z.string(), - provider: z.string().optional(), - recipient: z.record(z.string(), z.unknown()), - content: z.record(z.string(), z.unknown()), - variables: variablesSchema.optional(), - webhook_url: z.url(), + ...notificationPayloadFields, status: z.enum(NOTIFICATION_STATUS), - scheduled_at: z.coerce.date().optional(), error_message: z.string().optional(), retry_count: z.number().int().min(0), // Recovery claiming fields for horizontal scalability @@ -341,18 +356,9 @@ export const telegramConfigSchema = z.object({ export const baseNotificationRequestSchema = z .object({ request_id: UUIDV4Schema, - client_id: UUIDV4Schema, - client_name: z.string().optional(), - template_id: z.array(z.string().nullable().optional()).optional(), - channel: z.array(z.string()).min(1, "At least one channel is required."), - provider: z - .union([z.string(), z.array(z.string().nullable().optional())]) - .optional(), + ...notificationRequestFields, recipient: z.record(z.string(), z.unknown()), - content: z.record(z.string(), z.record(z.string(), z.string())).optional(), variables: variablesSchema.optional(), - scheduled_at: z.coerce.date().optional(), - webhook_url: z.url(), }) .superRefine((data, ctx) => { if ( @@ -375,14 +381,7 @@ export const baseNotificationRequestSchema = z */ export const baseBatchNotificationRequestSchema = z .object({ - client_id: UUIDV4Schema, - client_name: z.string().optional(), - channel: z.array(z.string()).min(1, "At least one channel is required."), - template_id: z.array(z.string().nullable().optional()).optional(), - provider: z - .union([z.string(), z.array(z.string().nullable().optional())]) - .optional(), - content: z.record(z.string(), z.record(z.string(), z.string())).optional(), + ...notificationRequestFields, recipients: z.array( z.looseObject({ request_id: UUIDV4Schema, @@ -390,8 +389,6 @@ export const baseBatchNotificationRequestSchema = z variables: variablesSchema.optional(), }), ).min(1, "At least one recipient is required."), - scheduled_at: z.coerce.date().optional(), - webhook_url: z.url(), }) .refine( (data) => { @@ -429,19 +426,11 @@ export const baseBatchNotificationRequestSchema = z Notification Template Request Schema */ export const notificationTemplateRequestSchema = z.object({ - name: z.string(), + ...notificationTemplateFields, template_id: z.string().optional(), - description: z.string().optional(), - content: z.record(z.string(), z.unknown()), - package: z.string(), }); -export const notificationTemplateUpdateRequestSchema = z.object({ - name: z.string(), - description: z.string().optional(), - content: z.record(z.string(), z.unknown()), - package: z.string(), -}); +export const notificationTemplateUpdateRequestSchema = z.object(notificationTemplateFields); // ============================================================================ // VALIDATION FUNCTIONS diff --git a/src/types/types.ts b/src/types/types.ts index 8a36f75..39449c9 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -63,6 +63,39 @@ export enum ALERT_TYPE { orphaned_pending = "orphaned_pending" } +export type RefillInterval = 'second' | 'minute' | 'hour' | 'day'; + +export interface DeliveryResult { + success: boolean; + messageId?: string; + providerResponse?: unknown; + error?: { + code: string; + message: string; + retryable: boolean; + }; +} + +export interface ProviderConfig { + id: string; + credentials: Record; + options?: ProviderOptions; +} + +export interface SimpleNSProvider { + readonly manifest: ProviderManifest; + + getNotificationSchema(): z.ZodSchema; + getRecipientSchema(): z.ZodObject>; + getContentSchema(): z.ZodObject>; + getRateLimitConfig(): RateLimitConfig; + + initialize(config: ProviderConfig): Promise; + healthCheck(): Promise; + send(notification: TNotification): Promise; + shutdown(): Promise; +} + // ============================================================================ // SCHEMA IMPORTS // ============================================================================ @@ -117,6 +150,25 @@ export type status_outbox = z.infer; export type base_notification = z.infer; +export type BaseNotification = Omit & { + notification_id: string; +}; +export type ProviderManifest = provider_manifest; +export type RateLimitConfig = { + maxTokens: number; + refillRate: number; + refillInterval?: RefillInterval; +}; + +export interface RateLimitResult { + allowed: boolean; + remainingTokens: number; + retryAfterMs?: number; + queuePosition?: number; +} + +export type HealthChecker = () => Promise; + // ============================================================================ // ADMIN NOTIFICATION CHANNEL TYPES // ============================================================================ @@ -140,3 +192,267 @@ export type provider_manifest = z.infer; export type plugin_document = z.infer; export type provider_document = z.infer; export type channel_routing_document = z.infer; + +export interface KafkaConsumerState { + consumer: import('kafkajs').Consumer | null; + isConsuming: boolean; +} + +export interface OutboxCronState { + pollIntervalId: NodeJS.Timeout | null; + cleanupIntervalId: NodeJS.Timeout | null; + statusPollIntervalId: NodeJS.Timeout | null; + isPolling: boolean; + isCleaningUp: boolean; + isPollingStatus: boolean; + shouldStop: boolean; +} + +export interface RecoveryCronState { + intervalId: NodeJS.Timeout | null; + isRunning: boolean; + shouldStop: boolean; + healthChecker: HealthChecker | null; + consecutiveFailures: number; +} + +export interface ChannelResult { + success: boolean; + error?: string; +} + +export interface AlertMetadata { + alertType?: AdminAlertType; + severity?: 'info' | 'warning' | 'critical'; + timestamp?: Date; + notificationId?: string; + channel?: string; + errorMessage?: string; +} + +export interface CredentialField { + name: string; + type: 'string' | 'url' | 'secret'; + label: string; + placeholder?: string; + description?: string; + required: boolean; + pattern?: string; +} + +export interface AdminChannelProvider { + readonly channelType: AdminChannelType; + readonly displayName: string; + send(message: string, metadata?: AlertMetadata): Promise; + testConnection(): Promise; + getCredentialSchema(): CredentialField[]; +} + +export interface AdminChannelMeta { + channelType: AdminChannelType; + displayName: string; + credentialFields: CredentialField[]; +} + +export interface RegisteredProvider { + provider: SimpleNSProvider; + id: string; + priority: number; +} + +export interface ChannelConfig { + default: string; + fallback?: string | string[]; +} + +export interface FieldDefinition { + name: string; + type: 'string' | 'email' | 'phone' | 'text' | 'number' | 'boolean'; + required: boolean; + description?: string; +} + +export interface ProviderMetadata { + id: string; + name: string; + displayName: string; + description?: string; + priority: number; + recipientFields: FieldDefinition[]; + contentFields: FieldDefinition[]; +} + +export interface ChannelMetadata { + providers: ProviderMetadata[]; + default?: string; + fallback?: string | string[]; +} + +export interface PluginMetadata { + channels: Record; +} + +export interface ProviderRateLimitOptions { + maxTokens?: number; + refillRate?: number; + refillInterval?: RefillInterval; +} + +export interface ProviderOptions { + priority?: number; + rateLimit?: ProviderRateLimitOptions; + [key: string]: unknown; +} + +export interface ProviderEntryBase { + package: string; + id: string; + credentials?: Record; + optionalConfig?: Record; + options?: ProviderOptions; +} + +export interface YamlProviderEntry extends ProviderEntryBase { + version?: string; +} + +export interface YamlConfig { + providers?: YamlProviderEntry[]; + channels?: Record; +} + +export interface ProviderEntry extends ProviderEntryBase { + credentials: Record; +} + +export interface SimpleNSConfig { + providers: ProviderEntry[]; + channels?: Record; +} + +export interface ProviderResponseDto { + _id?: string; + id: string; + plugin_name: string; + channel: string; + priority: number; + enabled: boolean; + options?: Record; + credentials_configured: boolean; + created_at?: Date; + updated_at?: Date; +} + +export type PluginSyncAction = + | 'PLUGIN_INSTALLED' + | 'PLUGIN_UNINSTALLED' + | 'PLUGIN_VERSION_CHANGED' + | 'PROVIDER_UPSERTED' + | 'PROVIDER_DELETED' + | 'CHANNEL_ROUTING_UPDATED' + | 'SYSTEM_RELOAD'; + +export interface PluginSyncPayload { + plugin_name?: string; + version?: string; + provider_id?: string; + channel?: string; + [key: string]: unknown; +} + +export interface PluginSyncMessage { + event_id: string; + source_instance_id: string; + timestamp: string; + action: PluginSyncAction; + payload: PluginSyncPayload; +} + +export type PluginSyncHandler = (message: PluginSyncMessage) => Promise; + +export interface DelayedEventWithRetries extends delayed_notification_topic { + _pollerRetries?: number; +} + +export interface IdempotencyRecord { + status: 'processing' | 'delivered' | 'failed' | 'rate_limited'; + retry_count: number; + updated_at: string; +} + +export interface StatusProcessResult { + dbUpdated: boolean; + webhookUrl?: string; + webhookPayload?: WebhookPayload; + notificationId?: string; +} + +export interface WebhookPayload { + request_id: string; + client_id: string; + notification_id: string; + status: string; + channel: string; + message: string; + occurred_at: string; +} + +export type ServiceContext = + | 'api' + | 'producer' + | 'consumer' + | 'cron' + | 'worker' + | 'delayedWorker' + | 'unifiedProcessor' + | 'redis' + | 'recoveryService' + | 'adminAlert' + | 'pluginLoader' + | 'pluginRegistry' + | 'rateLimiter'; + +export interface LogMeta { + notificationId?: string; + requestId?: string; + clientId?: string; + channel?: string; + workerId?: string; + topic?: string; + partition?: number; + [key: string]: unknown; +} + +export interface Logger { + info: (message: string, meta?: LogMeta) => void; + warn: (message: string, meta?: LogMeta) => void; + error: (message: string, meta?: LogMeta | unknown) => void; + debug: (message: string, meta?: LogMeta) => void; + success: (message: string, meta?: LogMeta) => void; +} + +export interface SendResult { + successCount: number; + failedCount: number; +} + +export interface ProcessingStats { + processed: number; + success: number; + failed: number; +} + +export interface ValidatedOutboxEntry { + _id: import('mongoose').Types.ObjectId; + notification_id: import('mongoose').Types.ObjectId | string; + topic: string; + payload: Record; + status: string; +} diff --git a/src/workers/consumers/status.consumer.ts b/src/workers/consumers/status.consumer.ts index 9b30f84..e45899e 100644 --- a/src/workers/consumers/status.consumer.ts +++ b/src/workers/consumers/status.consumer.ts @@ -1,6 +1,6 @@ -import { Consumer, EachMessagePayload } from "kafkajs"; +import { EachMessagePayload } from "kafkajs"; import { kafka } from "@src/config/kafka.config.js"; -import { CORE_TOPICS, NOTIFICATION_STATUS, NOTIFICATION_STATUS_SF, type notification_status_topic } from "@src/types/types.js"; +import { CORE_TOPICS, NOTIFICATION_STATUS, NOTIFICATION_STATUS_SF, type KafkaConsumerState, type notification_status_topic, type StatusProcessResult, type WebhookPayload } from "@src/types/types.js"; import { safeValidateNotificationStatusTopic } from "@src/types/schemas.js"; import notification_model from "@src/database/models/notification.models.js"; import { consumerLogger as logger } from "@src/workers/utils/logger.js"; @@ -11,12 +11,7 @@ const CONSUMER_GROUP_ID = "notification-status-group"; const WEBHOOK_TIMEOUT_MS = 30000; // 30 seconds // Consumer state management -interface ConsumerState { - consumer: Consumer | null; - isConsuming: boolean; -} - -const state: ConsumerState = { +const state: KafkaConsumerState = { consumer: null, isConsuming: false }; @@ -33,7 +28,7 @@ const mapToNotificationStatus = (externalStatus: NOTIFICATION_STATUS_SF): NOTIFI /** * Build webhook payload from status data */ -const buildWebhookPayload = (data: notification_status_topic) => ({ +const buildWebhookPayload = (data: notification_status_topic): WebhookPayload => ({ request_id: data.request_id, client_id: data.client_id, notification_id: data.notification_id.toString(), @@ -89,18 +84,11 @@ const sendWebhookCallback = ( /** * Result of processing a status message */ -interface ProcessResult { - dbUpdated: boolean; - webhookUrl?: string; - webhookPayload?: ReturnType; - notificationId?: string; -} - /** * Process a single status message * Returns result indicating if DB was updated and webhook info */ -const processStatusMessage = async ({ partition, message }: EachMessagePayload): Promise => { +const processStatusMessage = async ({ partition, message }: EachMessagePayload): Promise => { try { if (!message.value) { logger.warn("Received empty message, skipping"); diff --git a/src/workers/cron/background.cron.ts b/src/workers/cron/background.cron.ts index 878187a..fd7429e 100644 --- a/src/workers/cron/background.cron.ts +++ b/src/workers/cron/background.cron.ts @@ -1,5 +1,5 @@ import { env } from "@src/config/env.config.js"; -import { OUTBOX_STATUS } from "@src/types/types.js"; +import { OUTBOX_STATUS, type OutboxCronState } from "@src/types/types.js"; import outbox_model from "@src/database/models/outbox.models.js"; import status_outbox_model from "@src/database/models/status-outbox.models.js"; import { type status_outbox } from "@src/types/types.js"; @@ -8,17 +8,7 @@ import { cronLogger as logger } from "@src/workers/utils/logger.js"; import type { OutboxDocument } from "@src/workers/utils/validation.js"; // Cron state management -interface CronState { - pollIntervalId: NodeJS.Timeout | null; - cleanupIntervalId: NodeJS.Timeout | null; - statusPollIntervalId: NodeJS.Timeout | null; - isPolling: boolean; - isCleaningUp: boolean; - isPollingStatus: boolean; - shouldStop: boolean; -} - -const state: CronState = { +const state: OutboxCronState = { pollIntervalId: null, cleanupIntervalId: null, statusPollIntervalId: null, diff --git a/src/workers/recovery/recovery.cron.ts b/src/workers/recovery/recovery.cron.ts index 4d9754e..7751847 100644 --- a/src/workers/recovery/recovery.cron.ts +++ b/src/workers/recovery/recovery.cron.ts @@ -13,7 +13,7 @@ import mongoose from 'mongoose'; import { env } from '@src/config/env.config.js'; -import { NOTIFICATION_STATUS, ALERT_TYPE } from '@src/types/types.js'; +import { NOTIFICATION_STATUS, ALERT_TYPE, type HealthChecker, type RecoveryCronState } from '@src/types/types.js'; import notification_model from '@src/database/models/notification.models.js'; import alert_model from '@src/database/models/alert.models.js'; import status_outbox_model from '@src/database/models/status-outbox.models.js'; @@ -22,21 +22,10 @@ import { recoveryLogger as logger } from '@src/workers/utils/logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; import { NOTIFICATION_STATUS_SF } from '@src/types/types.js'; -// Health checker function type -type HealthChecker = () => Promise; - -// Cron state management -interface CronState { - intervalId: NodeJS.Timeout | null; - isRunning: boolean; - shouldStop: boolean; - healthChecker: HealthChecker | null; - consecutiveFailures: number; -} - const MAX_CONSECUTIVE_FAILURES = 5; -const state: CronState = { +// Cron state management +const state: RecoveryCronState = { intervalId: null, isRunning: false, shouldStop: false, diff --git a/src/workers/utils/logger.ts b/src/workers/utils/logger.ts index f753d7e..349640a 100644 --- a/src/workers/utils/logger.ts +++ b/src/workers/utils/logger.ts @@ -6,6 +6,7 @@ import winston from 'winston'; import LokiTransport from 'winston-loki'; import { env } from '@src/config/env.config.js'; +import type { LogMeta, Logger, ServiceContext } from '@src/types/types.js'; // Service context types const SERVICE_LABELS = { @@ -24,20 +25,6 @@ const SERVICE_LABELS = { rateLimiter: 'rate-limiter' } as const; -type ServiceContext = keyof typeof SERVICE_LABELS; - -// Log metadata interface -interface LogMeta { - notificationId?: string; - requestId?: string; - clientId?: string; - channel?: string; - workerId?: string; - topic?: string; - partition?: number; - [key: string]: unknown; -} - // Custom log levels including 'success' const customLevels = { levels: { @@ -175,17 +162,6 @@ const createWinstonLogger = (service: ServiceContext) => { }); }; -/** - * Logger interface matching our usage patterns - */ -export interface Logger { - info: (message: string, meta?: LogMeta) => void; - warn: (message: string, meta?: LogMeta) => void; - error: (message: string, meta?: LogMeta | unknown) => void; - debug: (message: string, meta?: LogMeta) => void; - success: (message: string, meta?: LogMeta) => void; -} - /** * Create a typed logger for a service context */ diff --git a/src/workers/utils/types.ts b/src/workers/utils/types.ts index 2304527..9602a07 100644 --- a/src/workers/utils/types.ts +++ b/src/workers/utils/types.ts @@ -1,14 +1 @@ -/** - * Common types used across worker modules - */ - -export interface SendResult { - successCount: number; - failedCount: number; -} - -export interface ProcessingStats { - processed: number; - success: number; - failed: number; -} +export type { ProcessingStats, SendResult } from '@src/types/types.js'; diff --git a/src/workers/utils/validation.ts b/src/workers/utils/validation.ts index 4315795..c6b6eaf 100644 --- a/src/workers/utils/validation.ts +++ b/src/workers/utils/validation.ts @@ -5,7 +5,7 @@ */ import mongoose from "mongoose"; -import { type outbox } from "@src/types/types.js"; +import { type outbox, type ValidatedOutboxEntry } from "@src/types/types.js"; import { safeValidateOutbox, safeValidateBaseNotification, @@ -14,6 +14,8 @@ import { import type { HydratedDocument } from "mongoose"; import { producerLogger as logger } from "./logger.js"; +export type { ValidatedOutboxEntry } from '@src/types/types.js'; + // Type for outbox document from MongoDB export type OutboxDocument = HydratedDocument; @@ -21,14 +23,6 @@ export type OutboxDocument = HydratedDocument; export type ValidPayload = Record; // Validated outbox entry -export interface ValidatedOutboxEntry { - _id: mongoose.Types.ObjectId; - notification_id: mongoose.Types.ObjectId | string; - topic: string; - payload: ValidPayload; - status: string; -} - /** * Validate an outbox entry and its payload * @returns ValidatedOutboxEntry if valid, null if validation fails From fcb0fdd738da510bf626cb747dfa49fed2830e69 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Tue, 15 Sep 2026 02:19:41 +0530 Subject: [PATCH 05/46] feat: enhance provider options with rate limiting and additional configuration support --- .../plugins/channel-routing-tab.tsx | 4 +- .../components/plugins/providers-tab.tsx | 167 +++++++++++++++++- dashboard/lib/api-client.ts | 6 +- dashboard/lib/types.ts | 11 +- src/api/controllers/providers.controller.ts | 6 +- src/database/models/provider.models.ts | 7 +- src/plugins/bootstrap/yaml-migrator.ts | 1 - src/plugins/loader/loader.ts | 6 +- .../services/provider-manager.service.ts | 39 ++-- src/types/schemas.ts | 23 ++- src/types/types.ts | 35 +--- .../plugins/provider-manager.service.test.ts | 7 +- 12 files changed, 231 insertions(+), 81 deletions(-) diff --git a/dashboard/components/plugins/channel-routing-tab.tsx b/dashboard/components/plugins/channel-routing-tab.tsx index 88e1915..6b00a19 100644 --- a/dashboard/components/plugins/channel-routing-tab.tsx +++ b/dashboard/components/plugins/channel-routing-tab.tsx @@ -275,7 +275,7 @@ export function ChannelRoutingTab() { {channelProviders.map((p) => ( - {p.id} (Priority: {p.priority}) + {p.id} (Priority: {p.options?.priority ?? 0}) ))} @@ -327,7 +327,7 @@ export function ChannelRoutingTab() { .filter((p) => p.id !== defaultProviderId && !fallbackIds.includes(p.id)) .map((p) => ( - {p.id} (Priority: {p.priority}) + {p.id} (Priority: {p.options?.priority ?? 0}) ))} diff --git a/dashboard/components/plugins/providers-tab.tsx b/dashboard/components/plugins/providers-tab.tsx index 89435b1..b949d67 100644 --- a/dashboard/components/plugins/providers-tab.tsx +++ b/dashboard/components/plugins/providers-tab.tsx @@ -50,6 +50,20 @@ import { } from "lucide-react"; import { toast } from "sonner"; +type AdditionalOption = { + id: string; + key: string; + value: string; +}; + +let additionalOptionId = 0; + +const createAdditionalOption = (key = "", value = ""): AdditionalOption => ({ + id: `additional-option-${additionalOptionId++}`, + key, + value, +}); + export function ProvidersTab() { const { data: providersData, @@ -69,6 +83,12 @@ export function ProvidersTab() { const [providerId, setProviderId] = useState(""); const [pluginName, setPluginName] = useState(""); const [priority, setPriority] = useState(0); + const [rateLimit, setRateLimit] = useState({ + maxTokens: "", + refillRate: "", + refillInterval: "second", + }); + const [additionalOptions, setAdditionalOptions] = useState([]); const [enabled, setEnabled] = useState(true); const [credentials, setCredentials] = useState>({}); const [saving, setSaving] = useState(false); @@ -90,6 +110,8 @@ export function ProvidersTab() { setProviderId(""); setPluginName(installedPlugins[0]?.name || ""); setPriority(0); + setRateLimit({ maxTokens: "", refillRate: "", refillInterval: "second" }); + setAdditionalOptions([]); setEnabled(true); setCredentials({}); setTestResult(null); @@ -107,13 +129,55 @@ export function ProvidersTab() { setEditingProvider(provider); setProviderId(provider.id); setPluginName(provider.plugin_name); - setPriority(provider.priority); + setPriority(provider.options?.priority ?? 0); + setRateLimit({ + maxTokens: provider.options?.rateLimit?.maxTokens?.toString() || "", + refillRate: provider.options?.rateLimit?.refillRate?.toString() || "", + refillInterval: provider.options?.rateLimit?.refillInterval || "second", + }); + setAdditionalOptions( + Object.entries(provider.options || {}) + .filter(([key]) => key !== "priority" && key !== "rateLimit") + .map(([key, value]) => ({ + ...createAdditionalOption( + key, + typeof value === "string" ? value : JSON.stringify(value), + ), + })) + ); setEnabled(provider.enabled); setCredentials({}); setTestResult(null); setModalOpen(true); }; + const buildProviderOptions = (): Record => { + const options: Record = { priority }; + const configuredRateLimit: Record = {}; + + if (rateLimit.maxTokens.trim() !== "") { + configuredRateLimit.maxTokens = parseInt(rateLimit.maxTokens, 10); + } + if (rateLimit.refillRate.trim() !== "") { + configuredRateLimit.refillRate = parseInt(rateLimit.refillRate, 10); + } + if (rateLimit.refillInterval) { + configuredRateLimit.refillInterval = rateLimit.refillInterval; + } + if (Object.keys(configuredRateLimit).length > 0) { + options.rateLimit = configuredRateLimit; + } + + for (const option of additionalOptions) { + const key = option.key.trim(); + if (!key || key === "priority" || key === "rateLimit") continue; + const value = option.value.trim(); + options[key] = /^-?\d+$/.test(value) ? parseInt(value, 10) : option.value; + } + + return options; + }; + const handleTestConnection = async () => { setTesting(true); setTestResult(null); @@ -144,7 +208,7 @@ export function ProvidersTab() { try { if (editingProvider) { await providerService.update(editingProvider.id, { - priority, + options: buildProviderOptions(), enabled, credentials: Object.keys(credentials).length > 0 ? credentials : undefined, }); @@ -158,7 +222,7 @@ export function ProvidersTab() { id: providerId.trim(), plugin_name: pluginName, credentials, - priority, + options: buildProviderOptions(), enabled, }); toast.success(`Provider '${providerId}' created`); @@ -301,7 +365,7 @@ export function ProvidersTab() {
Channel: {p.channel} - Priority: {p.priority} + Priority: {p.options?.priority || 0}
Encrypted (RSA-OAEP) @@ -423,6 +487,101 @@ export function ProvidersTab() {
+
+ +
+
+ + setRateLimit({ ...rateLimit, maxTokens: e.target.value })} + placeholder="100" + /> +
+
+ + setRateLimit({ ...rateLimit, refillRate: e.target.value })} + placeholder="10" + /> +
+
+ + +
+
+
+ +
+
+ + +
+ {additionalOptions.map((option, index) => ( +
+ { + const next = [...additionalOptions]; + next[index] = { ...next[index], key: e.target.value }; + setAdditionalOptions(next); + }} + /> + { + const next = [...additionalOptions]; + next[index] = { ...next[index], value: e.target.value }; + setAdditionalOptions(next); + }} + /> + +
+ ))} +
+ {/* Dynamic Credential Fields from Plugin Manifest */}
diff --git a/dashboard/lib/api-client.ts b/dashboard/lib/api-client.ts index 1dd95b7..a7e3194 100644 --- a/dashboard/lib/api-client.ts +++ b/dashboard/lib/api-client.ts @@ -148,15 +148,13 @@ export const providerService = { id: string; plugin_name: string; credentials: Record; - priority?: number; - options?: Record; + options?: Record & { priority?: number }; enabled?: boolean; }): Promise<{ message: string; provider: ProviderDto }> => apiClient.post('/api/providers', payload), update: (id: string, payload: { credentials?: Record; - priority?: number; - options?: Record; + options?: Record & { priority?: number }; enabled?: boolean; }): Promise<{ message: string; provider: ProviderDto }> => apiClient.put(`/api/providers/${id}`, payload), diff --git a/dashboard/lib/types.ts b/dashboard/lib/types.ts index 35126a8..377de20 100644 --- a/dashboard/lib/types.ts +++ b/dashboard/lib/types.ts @@ -249,9 +249,16 @@ export interface ProviderDto { id: string; plugin_name: string; channel: string; - priority: number; enabled: boolean; - options?: Record; + options?: { + priority?: number; + rateLimit?: { + maxTokens?: number; + refillRate?: number; + refillInterval?: "second" | "minute" | "hour" | "day"; + }; + [key: string]: unknown; + }; credentials_configured: boolean; decrypted_credentials?: Record; created_at?: string; diff --git a/src/api/controllers/providers.controller.ts b/src/api/controllers/providers.controller.ts index 5164366..97228b7 100644 --- a/src/api/controllers/providers.controller.ts +++ b/src/api/controllers/providers.controller.ts @@ -58,7 +58,7 @@ export const getProvider = async (req: Request, res: Response): Promise => */ export const createProvider = async (req: Request, res: Response): Promise => { try { - const { id, plugin_name, credentials, priority, options, enabled } = req.body; + const { id, plugin_name, credentials, options, enabled } = req.body; if (!id || typeof id !== 'string') { res.status(400).json({ @@ -88,7 +88,6 @@ export const createProvider = async (req: Request, res: Response): Promise id: id.trim(), plugin_name: plugin_name.trim(), credentials, - priority: typeof priority === 'number' ? priority : 0, options: options || {}, enabled: enabled !== false, }); @@ -114,11 +113,10 @@ export const createProvider = async (req: Request, res: Response): Promise */ export const updateProvider = async (req: Request, res: Response): Promise => { try { - const { credentials, priority, options, enabled } = req.body; + const { credentials, options, enabled } = req.body; const provider = await ProviderManagerService.updateProvider(req.params.id, { credentials, - priority: typeof priority === 'number' ? priority : undefined, options, enabled: typeof enabled === 'boolean' ? enabled : undefined, }); diff --git a/src/database/models/provider.models.ts b/src/database/models/provider.models.ts index 2659d3b..0b7e9d1 100644 --- a/src/database/models/provider.models.ts +++ b/src/database/models/provider.models.ts @@ -38,11 +38,6 @@ const provider_schema = new mongoose.Schema( required: true, index: true, }, - priority: { - type: Number, - default: 0, - index: true, - }, enabled: { type: Boolean, default: true, @@ -66,7 +61,7 @@ const provider_schema = new mongoose.Schema( ); // Compound index for priority-based provider routing per channel -provider_schema.index({ channel: 1, priority: -1 }); +provider_schema.index({ channel: 1, 'options.priority': -1 }); const provider_model = mongoose.models.Provider || mongoose.model('Provider', provider_schema); diff --git a/src/plugins/bootstrap/yaml-migrator.ts b/src/plugins/bootstrap/yaml-migrator.ts index 4a07cfa..d2661cd 100644 --- a/src/plugins/bootstrap/yaml-migrator.ts +++ b/src/plugins/bootstrap/yaml-migrator.ts @@ -181,7 +181,6 @@ export class YamlMigrator { id: entry.id, plugin_name: entry.package, channel, - priority: entry.options?.priority ?? 0, enabled: true, credentials: encryptedCreds, options: mergedOptions, diff --git a/src/plugins/loader/loader.ts b/src/plugins/loader/loader.ts index 57120e9..300e6d6 100644 --- a/src/plugins/loader/loader.ts +++ b/src/plugins/loader/loader.ts @@ -18,7 +18,7 @@ import Provider from '@src/database/models/provider.models.js'; import ChannelRouting from '@src/database/models/channel-routing.models.js'; import { ProviderManagerService } from '@src/plugins/services/provider-manager.service.js'; import { installNpmPackage } from './plugin-fs.js'; -import type { provider_document, ProviderEntry, SimpleNSConfig } from '@src/types/types.js'; +import type { provider_document, ProviderEntry, ProviderOptions, SimpleNSConfig } from '@src/types/types.js'; // Plugins directory for user-installed plugins const PLUGINS_DIR = join(process.cwd(), '.plugins'); @@ -404,7 +404,7 @@ export async function registerProvider( provider: SimpleNSProvider, id: string, credentials: Record, - options?: Record + options?: ProviderOptions ): Promise { await provider.initialize({ id, @@ -412,7 +412,7 @@ export async function registerProvider( options, }); - const priority = (options?.priority as number) || 0; + const priority = options?.priority ?? 0; PluginRegistry.register(provider, id, priority); } diff --git a/src/plugins/services/provider-manager.service.ts b/src/plugins/services/provider-manager.service.ts index c10aea4..aae8c67 100644 --- a/src/plugins/services/provider-manager.service.ts +++ b/src/plugins/services/provider-manager.service.ts @@ -15,7 +15,7 @@ import { import { importAndInstantiateProvider } from '@src/plugins/loader/plugin-fs.js'; import { PluginRegistry } from '@src/plugins/loader/registry.js'; import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; -import type { provider_document, ProviderConfig, ProviderResponseDto } from '@src/types/types.js'; +import type { provider_document, ProviderConfig, ProviderOptions, ProviderResponseDto } from '@src/types/types.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; export type { ProviderResponseDto } from '@src/types/types.js'; @@ -28,8 +28,7 @@ export class ProviderManagerService { id: string; plugin_name: string; credentials: Record; - priority?: number; - options?: Record; + options?: ProviderOptions; enabled?: boolean; }): Promise { logger.info(`Creating provider '${data.id}' using plugin '${data.plugin_name}'...`); @@ -50,7 +49,6 @@ export class ProviderManagerService { id: data.id, plugin_name: data.plugin_name, channel: plugin.manifest.channel, - priority: data.priority ?? 0, enabled: data.enabled !== false, credentials: encryptedCredentials, options: data.options || {}, @@ -78,9 +76,8 @@ export class ProviderManagerService { id: providerDoc.id, plugin_name: providerDoc.plugin_name, channel: providerDoc.channel, - priority: providerDoc.priority, enabled: providerDoc.enabled, - options: providerDoc.options as Record | undefined, + options: providerDoc.options as ProviderOptions | undefined, credentials_configured: true, created_at: providerDoc.created_at, updated_at: providerDoc.updated_at, @@ -94,8 +91,7 @@ export class ProviderManagerService { id: string, updates: { credentials?: Record; - priority?: number; - options?: Record; + options?: ProviderOptions; enabled?: boolean; } ): Promise { @@ -106,9 +102,6 @@ export class ProviderManagerService { throw new Error(`Provider '${id}' not found.`); } - if (updates.priority !== undefined) { - providerDoc.priority = updates.priority; - } if (updates.enabled !== undefined) { providerDoc.enabled = updates.enabled; } @@ -144,7 +137,6 @@ export class ProviderManagerService { id: providerDoc.id, plugin_name: providerDoc.plugin_name, channel: providerDoc.channel, - priority: providerDoc.priority, enabled: providerDoc.enabled, options: providerDoc.options as Record | undefined, credentials_configured: true, @@ -188,16 +180,15 @@ export class ProviderManagerService { * List all providers without exposing ciphertext credentials */ public static async listProviders(): Promise { - const providers = await Provider.find().sort({ channel: 1, priority: -1 }).lean(); + const providers = await Provider.find().sort({ channel: 1, 'options.priority': -1 }).lean(); return providers.map((p) => ({ _id: p._id.toString(), id: p.id, plugin_name: p.plugin_name, channel: p.channel, - priority: p.priority, enabled: p.enabled, - options: p.options as Record | undefined, + options: p.options as ProviderOptions | undefined, credentials_configured: !!p.credentials?.encrypted_data, created_at: p.created_at, updated_at: p.updated_at, @@ -224,9 +215,8 @@ export class ProviderManagerService { id: providerDoc.id, plugin_name: providerDoc.plugin_name, channel: providerDoc.channel, - priority: providerDoc.priority, enabled: providerDoc.enabled, - options: providerDoc.options as Record | undefined, + options: providerDoc.options as ProviderOptions | undefined, credentials_configured: !!providerDoc.credentials?.encrypted_data, decrypted_credentials: decrypted, created_at: providerDoc.created_at, @@ -241,11 +231,11 @@ export class ProviderManagerService { provider_id?: string; plugin_name?: string; credentials?: Record; - options?: Record; + options?: ProviderOptions; }): Promise<{ success: boolean; message: string }> { let pluginName = data.plugin_name; let credentials = data.credentials; - let options = data.options || {}; + let options: ProviderOptions = data.options || {}; if (data.provider_id) { const existing = await Provider.findOne({ id: data.provider_id }); @@ -253,7 +243,7 @@ export class ProviderManagerService { throw new Error(`Provider '${data.provider_id}' not found.`); } pluginName = existing.plugin_name; - options = { ...(existing.options as Record), ...options }; + options = { ...(existing.options as ProviderOptions), ...options }; if (!credentials || Object.keys(credentials).length === 0) { credentials = await decryptCredentials(existing.credentials); } @@ -303,8 +293,7 @@ export class ProviderManagerService { id: providerDoc.id, credentials, options: { - ...(providerDoc.options as Record), - priority: providerDoc.priority, + ...(providerDoc.options as ProviderOptions), }, }; @@ -316,7 +305,11 @@ export class ProviderManagerService { } } - PluginRegistry.registerOrReplace(providerInstance, providerDoc.id, providerDoc.priority ?? 0); + PluginRegistry.registerOrReplace( + providerInstance, + providerDoc.id, + providerDoc.options?.priority ?? 0 + ); logger.success(`Registered provider '${providerDoc.id}' in PluginRegistry.`); } diff --git a/src/types/schemas.ts b/src/types/schemas.ts index c103fa1..92b637c 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -288,6 +288,26 @@ export const providerManifestSchema = z.object({ optionalConfig: z.array(z.string()).optional(), }); + /** Options supplied by a YAML/JSON provider configuration entry. */ + export const providerConfigOptionsSchema = z.object({ + priority: z.number().int().optional(), + rateLimit: z.object({ + maxTokens: z.number().nonnegative().optional(), + refillRate: z.number().nonnegative().optional(), + refillInterval: z.enum(['second', 'minute', 'hour', 'day']).optional(), + }).optional(), + }).catchall(z.unknown()); + + /** Provider entry as read from YAML/JSON configuration. */ + export const providerConfigEntrySchema = z.object({ + package: z.string().min(1), + version: z.string().optional(), + id: z.string().min(1), + credentials: z.record(z.string(), z.string()).optional(), + optionalConfig: z.record(z.string(), z.string()).optional(), + options: providerConfigOptionsSchema.optional(), + }); + /** * Plugin document schema */ @@ -308,10 +328,9 @@ export const providerSchema = z.object({ id: z.string().min(1), plugin_name: z.string().min(1), channel: z.string().min(1), - priority: z.number().int().default(0), enabled: z.boolean().default(true), credentials: encryptedCredentialsSchema, - options: z.record(z.string(), z.unknown()).optional(), + options: providerConfigOptionsSchema.optional(), created_at: z.coerce.date().optional(), updated_at: z.coerce.date().optional(), }); diff --git a/src/types/types.ts b/src/types/types.ts index 39449c9..532e713 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -123,6 +123,8 @@ import { // Plugin & Provider schemas encryptedCredentialsSchema, providerManifestSchema, + providerConfigOptionsSchema, + providerConfigEntrySchema, pluginSchema, providerSchema, channelRoutingSchema, @@ -292,29 +294,11 @@ export interface PluginMetadata { channels: Record; } -export interface ProviderRateLimitOptions { - maxTokens?: number; - refillRate?: number; - refillInterval?: RefillInterval; -} - -export interface ProviderOptions { - priority?: number; - rateLimit?: ProviderRateLimitOptions; - [key: string]: unknown; -} +export type ProviderRateLimitOptions = z.infer['rateLimit']; +export type ProviderOptions = z.infer; -export interface ProviderEntryBase { - package: string; - id: string; - credentials?: Record; - optionalConfig?: Record; - options?: ProviderOptions; -} - -export interface YamlProviderEntry extends ProviderEntryBase { - version?: string; -} +export type ProviderConfigEntry = z.infer; +export type YamlProviderEntry = ProviderConfigEntry; export interface YamlConfig { providers?: YamlProviderEntry[]; @@ -325,9 +309,9 @@ export interface YamlConfig { }>; } -export interface ProviderEntry extends ProviderEntryBase { +export type ProviderEntry = ProviderConfigEntry & { credentials: Record; -} +}; export interface SimpleNSConfig { providers: ProviderEntry[]; @@ -342,9 +326,8 @@ export interface ProviderResponseDto { id: string; plugin_name: string; channel: string; - priority: number; enabled: boolean; - options?: Record; + options?: ProviderOptions; credentials_configured: boolean; created_at?: Date; updated_at?: Date; diff --git a/tests/unit/plugins/provider-manager.service.test.ts b/tests/unit/plugins/provider-manager.service.test.ts index 5dcdcbd..3a9837a 100644 --- a/tests/unit/plugins/provider-manager.service.test.ts +++ b/tests/unit/plugins/provider-manager.service.test.ts @@ -64,10 +64,9 @@ describe('ProviderManagerService', () => { id: 'mock-test', plugin_name: '@simplens/mock', channel: 'mock', - priority: 5, enabled: true, credentials: { encrypted_data: 'enc123' }, - options: {}, + options: { priority: 5 }, toObject: () => mockCreated, }; @@ -78,7 +77,7 @@ describe('ProviderManagerService', () => { id: 'mock-test', plugin_name: '@simplens/mock', credentials: {}, - priority: 5, + options: { priority: 5 }, }); expect(keypairManager.encryptCredentials).toHaveBeenCalled(); @@ -87,7 +86,7 @@ describe('ProviderManagerService', () => { id: 'mock-test', plugin_name: '@simplens/mock', channel: 'mock', - priority: 5, + options: { priority: 5 }, }) ); expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_UPSERTED', { From d54e9453ada38d3acd637128077839e02dc77d54 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Tue, 15 Sep 2026 02:33:57 +0530 Subject: [PATCH 06/46] feat: implement channel routing deletion and update provider management logic --- .../plugins/channel-routing-tab.tsx | 34 ++++++++++++- .../plugins/installed-plugins-tab.tsx | 2 +- .../components/plugins/providers-tab.tsx | 2 +- .../services/channel-routing.service.ts | 10 ++++ .../services/plugin-manager.service.ts | 41 ++++++++++++--- .../services/provider-manager.service.ts | 48 +++++++++++++---- .../plugins/plugin-manager.service.test.ts | 51 +++++++++++++++---- .../plugins/provider-manager.service.test.ts | 48 +++++++++++++---- 8 files changed, 196 insertions(+), 40 deletions(-) diff --git a/dashboard/components/plugins/channel-routing-tab.tsx b/dashboard/components/plugins/channel-routing-tab.tsx index 6b00a19..b415d0a 100644 --- a/dashboard/components/plugins/channel-routing-tab.tsx +++ b/dashboard/components/plugins/channel-routing-tab.tsx @@ -35,6 +35,7 @@ import { Loader2, AlertCircle, X, + Trash2, } from "lucide-react"; import { toast } from "sonner"; @@ -60,6 +61,7 @@ export function ChannelRoutingTab() { const [fallbackIds, setFallbackIds] = useState([]); const [partitions, setPartitions] = useState(6); const [saving, setSaving] = useState(false); + const [deletingChannel, setDeletingChannel] = useState(null); const allProviders = providersData?.providers || []; const routings = routingData?.routings || []; @@ -124,6 +126,22 @@ export function ChannelRoutingTab() { } }; + const handleDelete = async (route: ChannelRoutingDto) => { + if (!window.confirm(`Delete the routing for channel '${route.channel}'?`)) return; + + setDeletingChannel(route.channel); + try { + await channelRoutingService.delete(route.channel); + toast.success(`Routing for channel '${route.channel}' deleted`); + mutateRouting(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to delete channel routing"; + toast.error(message); + } finally { + setDeletingChannel(null); + } + }; + return (
{/* Header */} @@ -225,7 +243,7 @@ export function ChannelRoutingTab() {
-
+
+
diff --git a/dashboard/components/plugins/installed-plugins-tab.tsx b/dashboard/components/plugins/installed-plugins-tab.tsx index de90e2e..4630efe 100644 --- a/dashboard/components/plugins/installed-plugins-tab.tsx +++ b/dashboard/components/plugins/installed-plugins-tab.tsx @@ -344,7 +344,7 @@ export function InstalledPluginsTab() { Uninstall Plugin Are you sure you want to uninstall {pluginToUninstall}? This will delete the package from - .plugins/node_modules. Uninstallation will be blocked if any configured providers still depend on this plugin. + .plugins/node_modules. Any providers using this plugin and channel routings that reference those providers will also be deleted. diff --git a/dashboard/components/plugins/providers-tab.tsx b/dashboard/components/plugins/providers-tab.tsx index b949d67..f0ceaf5 100644 --- a/dashboard/components/plugins/providers-tab.tsx +++ b/dashboard/components/plugins/providers-tab.tsx @@ -682,7 +682,7 @@ export function ProvidersTab() { Delete Provider - Are you sure you want to delete {providerToDelete}? This will unregister the instance. + Are you sure you want to delete {providerToDelete}? This will unregister the instance and remove it from any channel routing. If it is a channel's primary provider, the first fallback will become primary. Deletion will be blocked if this provider is assigned in channel routing. diff --git a/src/plugins/services/channel-routing.service.ts b/src/plugins/services/channel-routing.service.ts index 0d25997..4a61e52 100644 --- a/src/plugins/services/channel-routing.service.ts +++ b/src/plugins/services/channel-routing.service.ts @@ -125,6 +125,11 @@ export class ChannelRoutingService { logger.info(`Deleting channel routing for '${channel}'...`); await ChannelRouting.deleteOne({ channel }); + PluginRegistry.setChannelConfig(channel, { + default: '', + fallback: [], + }); + await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { channel, }); @@ -147,6 +152,11 @@ export class ChannelRoutingService { default: routing.default_provider_id, fallback: routing.fallback_provider_ids || [], }); + } else { + PluginRegistry.setChannelConfig(channel, { + default: '', + fallback: [], + }); } }); } diff --git a/src/plugins/services/plugin-manager.service.ts b/src/plugins/services/plugin-manager.service.ts index aa72245..1613f50 100644 --- a/src/plugins/services/plugin-manager.service.ts +++ b/src/plugins/services/plugin-manager.service.ts @@ -7,6 +7,8 @@ import Plugin from '@src/database/models/plugin.models.js'; import Provider from '@src/database/models/provider.models.js'; +import ChannelRouting from '@src/database/models/channel-routing.models.js'; +import { PluginRegistry } from '@src/plugins/loader/registry.js'; import { installNpmPackage, uninstallNpmPackage, @@ -102,18 +104,11 @@ export class PluginManagerService { } /** - * Uninstall a plugin. Rejects if any configured providers depend on it. + * Uninstall a plugin and remove its dependent routing and provider records. */ public static async uninstallPlugin(packageName: string): Promise { logger.info(`Uninstalling plugin '${packageName}'...`); - const dependentProvidersCount = await Provider.countDocuments({ plugin_name: packageName }); - if (dependentProvidersCount > 0) { - throw new Error( - `Cannot uninstall plugin '${packageName}': ${dependentProvidersCount} provider(s) depend on it. Delete or reassign providers first.` - ); - } - const existing = await Plugin.findOne({ name: packageName }); if (!existing) { throw new Error(`Plugin '${packageName}' not found.`); @@ -122,6 +117,36 @@ export class PluginManagerService { await PluginSyncService.runWithLock(async () => { try { uninstallNpmPackage(packageName); + + const dependentProviders = await Provider.find({ plugin_name: packageName }) + .select({ id: 1 }) + .lean(); + const providerIds = dependentProviders.map((provider) => provider.id); + + if (providerIds.length > 0) { + const dependentRoutings = await ChannelRouting.find({ + $or: [ + { default_provider_id: { $in: providerIds } }, + { fallback_provider_ids: { $in: providerIds } }, + ], + }).select({ channel: 1 }).lean(); + const routingChannels = dependentRoutings.map((routing) => routing.channel); + + if (routingChannels.length > 0) { + await ChannelRouting.deleteMany({ channel: { $in: routingChannels } }); + for (const channel of routingChannels) { + PluginRegistry.setChannelConfig(channel, { default: '', fallback: [] }); + await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { channel }); + } + } + + await Provider.deleteMany({ plugin_name: packageName }); + for (const providerId of providerIds) { + PluginRegistry.unregister(providerId); + await PluginSyncService.publish('PROVIDER_DELETED', { provider_id: providerId }); + } + } + await Plugin.deleteOne({ name: packageName }); await PluginSyncService.publish('PLUGIN_UNINSTALLED', { diff --git a/src/plugins/services/provider-manager.service.ts b/src/plugins/services/provider-manager.service.ts index aae8c67..fcfa413 100644 --- a/src/plugins/services/provider-manager.service.ts +++ b/src/plugins/services/provider-manager.service.ts @@ -146,24 +146,52 @@ export class ProviderManagerService { } /** - * Delete a provider instance. Rejects if used in channel routing. + * Delete a provider instance and remove it from channel routing. */ public static async deleteProvider(id: string): Promise { logger.info(`Deleting provider '${id}'...`); - const routingUse = await ChannelRouting.findOne({ + const providerDoc = await Provider.findOne({ id }); + if (!providerDoc) { + throw new Error(`Provider '${id}' not found.`); + } + + const routings = await ChannelRouting.find({ $or: [{ default_provider_id: id }, { fallback_provider_ids: id }], - }); + }).lean(); - if (routingUse) { - throw new Error( - `Cannot delete provider '${id}': it is configured in channel '${routingUse.channel}' routing.` + for (const routing of routings) { + const remainingFallbacks = (routing.fallback_provider_ids || []).filter( + (providerId: string) => providerId !== id ); - } - const providerDoc = await Provider.findOne({ id }); - if (!providerDoc) { - throw new Error(`Provider '${id}' not found.`); + if (routing.default_provider_id === id) { + const [nextDefault, ...fallbacks] = remainingFallbacks; + if (nextDefault) { + await ChannelRouting.updateOne( + { channel: routing.channel }, + { + default_provider_id: nextDefault, + fallback_provider_ids: fallbacks, + } + ); + } else { + await ChannelRouting.deleteOne({ channel: routing.channel }); + } + } else { + await ChannelRouting.updateOne( + { channel: routing.channel }, + { fallback_provider_ids: remainingFallbacks } + ); + } + + PluginRegistry.setChannelConfig(routing.channel, { + default: remainingFallbacks[0] || '', + fallback: remainingFallbacks.slice(1), + }); + await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { + channel: routing.channel, + }); } await Provider.deleteOne({ id }); diff --git a/tests/unit/plugins/plugin-manager.service.test.ts b/tests/unit/plugins/plugin-manager.service.test.ts index 1f3750a..6cf5fe5 100644 --- a/tests/unit/plugins/plugin-manager.service.test.ts +++ b/tests/unit/plugins/plugin-manager.service.test.ts @@ -6,6 +6,8 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import { PluginManagerService } from '../../../src/plugins/services/plugin-manager.service.js'; import Plugin from '../../../src/database/models/plugin.models.js'; import Provider from '../../../src/database/models/provider.models.js'; +import ChannelRouting from '../../../src/database/models/channel-routing.models.js'; +import { PluginRegistry } from '../../../src/plugins/loader/registry.js'; import * as pluginFs from '../../../src/plugins/loader/plugin-fs.js'; import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; @@ -116,27 +118,56 @@ describe('PluginManagerService', () => { }); describe('uninstallPlugin', () => { - it('should reject uninstallation if providers depend on the plugin', async () => { - vi.spyOn(Provider, 'countDocuments').mockResolvedValue(2 as any); - - await expect(PluginManagerService.uninstallPlugin('@simplens/mock')).rejects.toThrow( - "Cannot uninstall plugin '@simplens/mock': 2 provider(s) depend on it." - ); - }); - - it('should uninstall and remove from DB when no providers depend on it', async () => { - vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + it('should remove dependent routings and providers before uninstalling the plugin', async () => { vi.spyOn(Plugin, 'findOne').mockResolvedValue({ name: '@simplens/mock' } as any); + vi.spyOn(Provider, 'find').mockReturnValue({ + select: () => ({ + lean: vi.fn().mockResolvedValue([{ id: 'mock-provider' }]), + }), + } as any); + vi.spyOn(ChannelRouting, 'find').mockReturnValue({ + select: () => ({ + lean: vi.fn().mockResolvedValue([{ channel: 'mock' }]), + }), + } as any); + const routingDeleteSpy = vi.spyOn(ChannelRouting, 'deleteMany').mockResolvedValue({ deletedCount: 1 } as any); + const providerDeleteSpy = vi.spyOn(Provider, 'deleteMany').mockResolvedValue({ deletedCount: 1 } as any); + const unregisterSpy = vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); const uninstallSpy = vi.spyOn(pluginFs, 'uninstallNpmPackage').mockReturnValue(undefined); const deleteSpy = vi.spyOn(Plugin, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); await PluginManagerService.uninstallPlugin('@simplens/mock'); expect(uninstallSpy).toHaveBeenCalledWith('@simplens/mock'); + expect(routingDeleteSpy).toHaveBeenCalledWith({ channel: { $in: ['mock'] } }); + expect(providerDeleteSpy).toHaveBeenCalledWith({ plugin_name: '@simplens/mock' }); + expect(unregisterSpy).toHaveBeenCalledWith('mock-provider'); expect(deleteSpy).toHaveBeenCalledWith({ name: '@simplens/mock' }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('CHANNEL_ROUTING_UPDATED', { + channel: 'mock', + }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_DELETED', { + provider_id: 'mock-provider', + }); expect(PluginSyncService.publish).toHaveBeenCalledWith('PLUGIN_UNINSTALLED', { plugin_name: '@simplens/mock', }); }); + + it('should uninstall and remove the plugin when no providers depend on it', async () => { + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ name: '@simplens/mock' } as any); + vi.spyOn(Provider, 'find').mockReturnValue({ + select: () => ({ + lean: vi.fn().mockResolvedValue([]), + }), + } as any); + const uninstallSpy = vi.spyOn(pluginFs, 'uninstallNpmPackage').mockReturnValue(undefined); + const deleteSpy = vi.spyOn(Plugin, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); + + await PluginManagerService.uninstallPlugin('@simplens/mock'); + + expect(uninstallSpy).toHaveBeenCalledWith('@simplens/mock'); + expect(deleteSpy).toHaveBeenCalledWith({ name: '@simplens/mock' }); + }); }); }); diff --git a/tests/unit/plugins/provider-manager.service.test.ts b/tests/unit/plugins/provider-manager.service.test.ts index 3a9837a..aafe446 100644 --- a/tests/unit/plugins/provider-manager.service.test.ts +++ b/tests/unit/plugins/provider-manager.service.test.ts @@ -97,25 +97,55 @@ describe('ProviderManagerService', () => { }); describe('deleteProvider', () => { - it('should reject deletion if provider is used in channel routing', async () => { - vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue({ - channel: 'email', - default_provider_id: 'gmail-prod', + it('should remove the provider from routing and promote the first fallback', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ id: 'gmail-prod' } as any); + vi.spyOn(ChannelRouting, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + channel: 'email', + default_provider_id: 'gmail-prod', + fallback_provider_ids: ['smtp-prod', 'backup-prod'], + }, + ]), } as any); - - await expect(ProviderManagerService.deleteProvider('gmail-prod')).rejects.toThrow( - "Cannot delete provider 'gmail-prod': it is configured in channel 'email' routing." + const updateSpy = vi.spyOn(ChannelRouting, 'updateOne').mockResolvedValue({ acknowledged: true } as any); + const deleteRoutingSpy = vi.spyOn(ChannelRouting, 'deleteOne').mockResolvedValue({ deletedCount: 0 } as any); + vi.spyOn(Provider, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); + vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); + + await ProviderManagerService.deleteProvider('gmail-prod'); + + expect(updateSpy).toHaveBeenCalledWith( + { channel: 'email' }, + { + default_provider_id: 'smtp-prod', + fallback_provider_ids: ['backup-prod'], + } ); + expect(deleteRoutingSpy).not.toHaveBeenCalled(); + expect(PluginSyncService.publish).toHaveBeenCalledWith('CHANNEL_ROUTING_UPDATED', { + channel: 'email', + }); }); - it('should delete from DB and unregister from registry if not in use', async () => { - vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue(null); + it('should delete the routing when the provider has no fallback replacement', async () => { vi.spyOn(Provider, 'findOne').mockResolvedValue({ id: 'mock-unused' } as any); + vi.spyOn(ChannelRouting, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + channel: 'mock', + default_provider_id: 'mock-unused', + fallback_provider_ids: [], + }, + ]), + } as any); + const deleteRoutingSpy = vi.spyOn(ChannelRouting, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); const deleteSpy = vi.spyOn(Provider, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); const unregisterSpy = vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); await ProviderManagerService.deleteProvider('mock-unused'); + expect(deleteRoutingSpy).toHaveBeenCalledWith({ channel: 'mock' }); expect(deleteSpy).toHaveBeenCalledWith({ id: 'mock-unused' }); expect(unregisterSpy).toHaveBeenCalledWith('mock-unused'); expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_DELETED', { From 5f341938b359ed2b9ff1d85c7d3b914d6c322152 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Tue, 15 Sep 2026 03:12:34 +0530 Subject: [PATCH 07/46] feat: enhance plugin management with credential caching and UI improvements --- dashboard/app/plugins/page.tsx | 39 ++------ .../plugins/channel-routing-tab.tsx | 38 ++++---- .../plugins/installed-plugins-tab.tsx | 92 +++++++++++++++---- .../components/plugins/providers-tab.tsx | 30 +++--- .../services/plugin-manager.service.ts | 2 + .../services/provider-manager.service.ts | 46 +++++++++- .../plugins/provider-manager.service.test.ts | 30 ++++++ 7 files changed, 199 insertions(+), 78 deletions(-) diff --git a/dashboard/app/plugins/page.tsx b/dashboard/app/plugins/page.tsx index d3b835b..d69b61e 100644 --- a/dashboard/app/plugins/page.tsx +++ b/dashboard/app/plugins/page.tsx @@ -2,7 +2,6 @@ import { useState } from "react"; import { DashboardLayout } from "@/components/layout/dashboard-layout"; -import { PageToolbar, PageToolbarSection, PageToolbarSpacer } from "@/components/ui/page-toolbar"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Puzzle, Server, Package, GitBranch } from "lucide-react"; import { ProvidersTab } from "@/components/plugins/providers-tab"; @@ -18,49 +17,31 @@ export default function PluginsPage() { description="Manage installed plugins and providers" >
- {/* Page Toolbar */} - - -
-
- -
-
-

Plugin & Provider Hub

-

- Dynamically manage provider plugins, encrypted credentials, and cascading routing. -

-
-
- - - - {/* 3-Tab Interface */} - - - - - Providers - + + Installed Plugins + + + Providers + Channel Routing - - - - + + + + diff --git a/dashboard/components/plugins/channel-routing-tab.tsx b/dashboard/components/plugins/channel-routing-tab.tsx index b415d0a..3c67156 100644 --- a/dashboard/components/plugins/channel-routing-tab.tsx +++ b/dashboard/components/plugins/channel-routing-tab.tsx @@ -200,37 +200,37 @@ export function ChannelRoutingTab() { ) : ( -
+
{routings.map((route) => ( - +
- + {route.channel}
-
- +
+ {route.partitions || 6} Partitions
{/* Routing Cascade Chain */}
-
+
Cascading Failover Route
-
- +
+ Primary: {route.default_provider_id} {route.fallback_provider_ids && route.fallback_provider_ids.length > 0 ? ( route.fallback_provider_ids.map((fId, idx) => ( -
- - +
+ + Fallback {idx + 1}: {fId}
@@ -243,20 +243,23 @@ export function ChannelRoutingTab() {
-
+
diff --git a/dashboard/components/plugins/installed-plugins-tab.tsx b/dashboard/components/plugins/installed-plugins-tab.tsx index 4630efe..5b8594a 100644 --- a/dashboard/components/plugins/installed-plugins-tab.tsx +++ b/dashboard/components/plugins/installed-plugins-tab.tsx @@ -28,7 +28,7 @@ import { AlertDialogTitle, } from "@/components/ui/alert-dialog"; import { Skeleton } from "@/components/ui/skeleton"; -import { Download, Package, RefreshCw, Trash2, ArrowUpDown, Loader2 } from "lucide-react"; +import { Download, Package, RefreshCw, Trash2, ArrowUpDown, Loader2, ExternalLink, Info } from "lucide-react"; import { toast } from "sonner"; export function InstalledPluginsTab() { @@ -50,6 +50,7 @@ export function InstalledPluginsTab() { const [uninstallDialogOpen, setUninstallDialogOpen] = useState(false); const [pluginToUninstall, setPluginToUninstall] = useState(null); const [uninstalling, setUninstalling] = useState(false); + const [detailsPlugin, setDetailsPlugin] = useState(null); const handleInstall = async (e: React.FormEvent) => { e.preventDefault(); @@ -175,10 +176,10 @@ export function InstalledPluginsTab() { ) : ( -
+
{plugins.map((plugin) => ( - - + +

{plugin.name}

@@ -203,20 +204,20 @@ export function InstalledPluginsTab() {
- {plugin.manifest?.description && ( -

- {plugin.manifest.description} -

- )} - - {plugin.manifest?.requiredCredentials && plugin.manifest.requiredCredentials.length > 0 && ( -
- Credentials required:{" "} - {plugin.manifest.requiredCredentials.join(", ")} -
- )} +

+ {plugin.manifest?.description || "No description provided."} +

-
+
+
+ {editingProvider && ( +

+ Leave credential fields empty to test with the encrypted credentials already stored for this provider. +

+ )} {selectedPlugin?.manifest?.requiredCredentials && selectedPlugin.manifest.requiredCredentials.length > 0 ? ( @@ -644,17 +649,19 @@ export function ProvidersTab() {
- + {editingProvider && ( + + )}
+ )} +
)) ) : ( From 71cab71208f1469f0d1584ba34462764b65cf71e Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Tue, 15 Sep 2026 07:40:48 +0530 Subject: [PATCH 09/46] feat: implement plugin catalog functionality with official and community options --- dashboard/app/plugins/page.tsx | 2 +- .../plugins/installed-plugins-tab.tsx | 175 +++++++++++++----- dashboard/lib/api-client.ts | 3 + dashboard/lib/types.ts | 7 + src/api/controllers/plugins.controller.ts | 35 ++++ src/api/routes/plugins.routes.ts | 2 + src/types/types.ts | 7 + 7 files changed, 181 insertions(+), 50 deletions(-) diff --git a/dashboard/app/plugins/page.tsx b/dashboard/app/plugins/page.tsx index d69b61e..ac8c97e 100644 --- a/dashboard/app/plugins/page.tsx +++ b/dashboard/app/plugins/page.tsx @@ -9,7 +9,7 @@ import { InstalledPluginsTab } from "@/components/plugins/installed-plugins-tab" import { ChannelRoutingTab } from "@/components/plugins/channel-routing-tab"; export default function PluginsPage() { - const [activeTab, setActiveTab] = useState("providers"); + const [activeTab, setActiveTab] = useState("plugins"); return ( ("official"); + const [catalogSearch, setCatalogSearch] = useState(""); + const [selectedVersions, setSelectedVersions] = useState>({}); + const [installingPackage, setInstallingPackage] = useState(null); + + const { data: officialCatalog = [], isLoading: officialLoading, error: officialError } = useSWR( + installModalOpen && catalogTab === "official" ? "plugin-catalog-official" : null, + () => pluginService.listCatalog("official") + ); + const { data: communityCatalog = [], isLoading: communityLoading, error: communityError } = useSWR( + installModalOpen && catalogTab === "community" ? "plugin-catalog-community" : null, + () => pluginService.listCatalog("community") + ); const [versionModalOpen, setVersionModalOpen] = useState(false); const [selectedPlugin, setSelectedPlugin] = useState(null); @@ -52,26 +64,34 @@ export function InstalledPluginsTab() { const [uninstalling, setUninstalling] = useState(false); const [detailsPlugin, setDetailsPlugin] = useState(null); - const handleInstall = async (e: React.FormEvent) => { - e.preventDefault(); - if (!packageName.trim()) return; - - setInstalling(true); + const handleInstall = async (plugin: PluginCatalogItem) => { + const version = selectedVersions[plugin.package] || "latest"; + setInstallingPackage(plugin.package); try { - await pluginService.install(packageName.trim(), packageVersion.trim() || undefined); - toast.success(`Plugin '${packageName}' installed successfully`); - setInstallModalOpen(false); - setPackageName(""); - setPackageVersion(""); + await pluginService.install(plugin.package, version === "latest" ? undefined : version); + toast.success(`Plugin '${plugin.name}' installed successfully`); mutate(); } catch (err: unknown) { const message = err instanceof Error ? err.message : "Failed to install plugin"; toast.error(message); } finally { - setInstalling(false); + setInstallingPackage(null); } }; + const catalog = catalogTab === "official" ? officialCatalog : communityCatalog; + const catalogLoading = catalogTab === "official" ? officialLoading : communityLoading; + const catalogError = catalogTab === "official" ? officialError : communityError; + const filteredCatalog = useMemo(() => { + const query = catalogSearch.trim().toLowerCase(); + if (!query) return catalog; + return catalog.filter((plugin) => + [plugin.name, plugin.package, plugin.description].some((value) => + value.toLowerCase().includes(query) + ) + ); + }, [catalog, catalogSearch]); + const handleChangeVersion = async (e: React.FormEvent) => { e.preventDefault(); if (!selectedPlugin || !targetVersion.trim()) return; @@ -110,7 +130,11 @@ export function InstalledPluginsTab() { } }; - const plugins = data?.plugins || []; + const plugins = useMemo(() => data?.plugins || [], [data?.plugins]); + const installedPackages = useMemo( + () => new Set(plugins.map((plugin) => plugin.name)), + [plugins] + ); return (
@@ -308,50 +332,103 @@ export function InstalledPluginsTab() { {/* Install Plugin Dialog */} - -
+ Install Plugin Package - Install a new notification provider package from npm into SimpleNS. + Browse official and community provider packages available for SimpleNS. -
-
- +
+
+ setPackageName(e.target.value)} - required - /> -
-
- - setPackageVersion(e.target.value)} + aria-label="Search plugins" + placeholder="Search by name, package, or description" + className="pl-9" + value={catalogSearch} + onChange={(event) => setCatalogSearch(event.target.value)} />
+ setCatalogTab(value as "official" | "community")}> + + Official + Community + + + {catalogLoading ? ( +
+ Loading plugins... +
+ ) : catalogError ? ( +
+ Failed to load {catalogTab} plugins. +
+ ) : filteredCatalog.length === 0 ? ( +
+ {catalogTab === "community" && catalog.length === 0 + ? "No community plugins are available yet." + : "No plugins match your search."} +
+ ) : ( +
+ {filteredCatalog.map((plugin) => { + const versions = plugin.versions?.length ? plugin.versions : ["latest"]; + const selectedVersion = selectedVersions[plugin.package] || versions[0]; + return ( +
+
+
{plugin.name}
+
{plugin.package}
+

{plugin.description}

+
+
+ + {installedPackages.has(plugin.package) ? ( + + ) : ( + + )} +
+
+ ); + })} +
+ )} + +
- - -
diff --git a/dashboard/lib/api-client.ts b/dashboard/lib/api-client.ts index a7e3194..68e93de 100644 --- a/dashboard/lib/api-client.ts +++ b/dashboard/lib/api-client.ts @@ -20,6 +20,7 @@ import { AdminChannelFormData, AdminChannelProviderMeta, InstalledPlugin, + PluginCatalogItem, ProviderDto, ChannelRoutingDto, } from './types'; @@ -130,6 +131,8 @@ export const alertService = { }; export const pluginService = { + listCatalog: (category: 'official' | 'community'): Promise => + apiClient.get(`/api/plugins/catalog/${category}`), getMetadata: (): Promise => apiClient.get('/api/plugins'), listInstalled: (): Promise<{ plugins: InstalledPlugin[] }> => apiClient.get('/api/plugins/installed'), install: (packageName: string, version?: string): Promise<{ message: string; plugin: InstalledPlugin }> => diff --git a/dashboard/lib/types.ts b/dashboard/lib/types.ts index 377de20..f086b53 100644 --- a/dashboard/lib/types.ts +++ b/dashboard/lib/types.ts @@ -244,6 +244,13 @@ export interface InstalledPlugin { updated_at?: string; } +export interface PluginCatalogItem { + name: string; + package: string; + description: string; + versions?: string[]; +} + export interface ProviderDto { _id?: string; id: string; diff --git a/src/api/controllers/plugins.controller.ts b/src/api/controllers/plugins.controller.ts index 5275d26..7d08e6e 100644 --- a/src/api/controllers/plugins.controller.ts +++ b/src/api/controllers/plugins.controller.ts @@ -7,6 +7,10 @@ import { Request, Response } from 'express'; import { PluginRegistry, PluginManagerService } from '@src/plugins/index.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; +import axios from 'axios'; +import type { plugin_catalog_entry } from '@src/types/types.js'; + +const PLUGIN_CATALOG_BASE_URL = 'https://www.simplens.in/plugins'; /** * GET /api/plugins @@ -50,6 +54,37 @@ export const listInstalledPlugins = async (_req: Request, res: Response): Promis } }; +/** + * GET /api/plugins/catalog/:category + * Proxy the public plugin catalog server-side to avoid browser CORS restrictions. + */ +export const listPluginCatalog = async (req: Request, res: Response): Promise => { + const category = req.params.category; + if (category !== 'official' && category !== 'community') { + res.status(400).json({ + error: 'Bad Request', + message: 'Catalog category must be official or community', + }); + return; + } + + try { + const response = await axios.get( + `${PLUGIN_CATALOG_BASE_URL}/${category}`, + { timeout: 10000 } + ); + const catalog = Array.isArray(response.data) ? response.data : []; + res.json(catalog); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error loading ${category} plugin catalog: ${message}`); + res.status(502).json({ + error: 'Bad Gateway', + message: `Failed to load ${category} plugin catalog`, + }); + } +}; + /** * POST /api/plugins/install * Install an npm plugin package diff --git a/src/api/routes/plugins.routes.ts b/src/api/routes/plugins.routes.ts index afdf88c..67d379c 100644 --- a/src/api/routes/plugins.routes.ts +++ b/src/api/routes/plugins.routes.ts @@ -8,6 +8,7 @@ import { Router } from 'express'; import { getPluginsMetadata, listInstalledPlugins, + listPluginCatalog, installPlugin, changePluginVersion, uninstallPlugin, @@ -17,6 +18,7 @@ const router = Router(); router.get('/', getPluginsMetadata); router.get('/installed', listInstalledPlugins); +router.get('/catalog/:category', listPluginCatalog); router.post('/install', installPlugin); router.put('/version', changePluginVersion); router.delete('/:scope/:package', uninstallPlugin); diff --git a/src/types/types.ts b/src/types/types.ts index 532e713..4cfc588 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -298,6 +298,13 @@ export type ProviderRateLimitOptions = z.infer; export type ProviderConfigEntry = z.infer; + +export type plugin_catalog_entry = { + name: string; + package: string; + description: string; + versions?: string[]; +}; export type YamlProviderEntry = ProviderConfigEntry; export interface YamlConfig { From cc5952e87d6e673d2154ba064182eb6c9d7bd979 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Tue, 15 Sep 2026 07:45:22 +0530 Subject: [PATCH 10/46] feat: enforce centralized type and schema declarations in TypeScript --- AGENTS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 083f22a..d943481 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,9 @@ Use TypeScript with strict typing and existing `@src/*` path aliases. Match curr Plan changes before writing code. Follow low-level design principles: single responsibility, clear interfaces, useful dependency inversion, and explicit error handling. Do not duplicate logic; extract shared behavior into focused utilities, services, or test helpers. Always use `axios` instead of the plain `fetch` API in this project. +## Type & Schema Centralisation +All TypeScript interfaces and types that live inside `src/` **must** be declared in `src/types/types.ts`. All Zod (or other validation library) schemas must be declared in `src/types/schemas.ts`. Individual source files must **not** define their own local interfaces, types, or schemas — import everything from `@src/types/types` or `@src/types/schemas` instead. This rule applies to every layer: API controllers, services, plugins, workers, processors, utilities, and database modules. + ## Testing Guidelines Vitest is the test runner. Integration tests also use `supertest`, `mongodb-memory-server`, and Redis mocks. Name test files with `.test.ts`, for example `tests/unit/plugins/loader.test.ts`. From 1fb3f881b53c3c4a02492d6d698a8bcfa770dd86 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Tue, 15 Sep 2026 07:47:23 +0530 Subject: [PATCH 11/46] fix: remove unused import from plugins page --- dashboard/app/plugins/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/app/plugins/page.tsx b/dashboard/app/plugins/page.tsx index ac8c97e..575e6a5 100644 --- a/dashboard/app/plugins/page.tsx +++ b/dashboard/app/plugins/page.tsx @@ -3,7 +3,7 @@ import { useState } from "react"; import { DashboardLayout } from "@/components/layout/dashboard-layout"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { Puzzle, Server, Package, GitBranch } from "lucide-react"; +import { Server, Package, GitBranch } from "lucide-react"; import { ProvidersTab } from "@/components/plugins/providers-tab"; import { InstalledPluginsTab } from "@/components/plugins/installed-plugins-tab"; import { ChannelRoutingTab } from "@/components/plugins/channel-routing-tab"; From 26ed80865994d021deef7df366f21e3df7d450f7 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Wed, 16 Sep 2026 18:40:18 +0530 Subject: [PATCH 12/46] refactor: streamline plugin management and enhance configuration handling --- src/api/utils/utils.ts | 2 +- src/plugins/bootstrap/yaml-migrator.ts | 63 +---- src/plugins/index.ts | 1 - src/plugins/interfaces/provider.types.ts | 12 - src/plugins/loader/loader.ts | 250 ++---------------- src/plugins/loader/plugin-fs.ts | 74 +++++- src/plugins/loader/registry.ts | 8 - src/plugins/loader/router.ts | 18 +- .../services/plugin-manager.service.ts | 85 +++--- .../services/provider-manager.service.ts | 99 ++++--- src/plugins/sync/plugin-sync.service.ts | 4 - src/processors/unified/unified.consumer.ts | 2 +- 12 files changed, 201 insertions(+), 417 deletions(-) delete mode 100644 src/plugins/interfaces/provider.types.ts diff --git a/src/api/utils/utils.ts b/src/api/utils/utils.ts index 6c38d2a..47fd938 100644 --- a/src/api/utils/utils.ts +++ b/src/api/utils/utils.ts @@ -13,6 +13,7 @@ import { getTopicForChannel, CORE_TOPICS, delayed_notification_topic, + SimpleNSProvider, } from "@src/types/types.js"; import { NOTIFICATION_STATUS, OUTBOX_STATUS } from "@src/types/types.js"; import mongoose from "mongoose"; @@ -21,7 +22,6 @@ import outbox_model from "@src/database/models/outbox.models.js"; import { apiLogger as logger } from "@src/workers/utils/logger.js"; import { PluginRegistry } from "@src/plugins/index.js"; import notification_template_model from "@src/database/models/notification-template.models.js"; -import { SimpleNSProvider } from "@src/plugins/interfaces/provider.types.js"; import { ZodError } from "zod"; const preloadTemplatesByIds = async ( diff --git a/src/plugins/bootstrap/yaml-migrator.ts b/src/plugins/bootstrap/yaml-migrator.ts index d2661cd..48ff56b 100644 --- a/src/plugins/bootstrap/yaml-migrator.ts +++ b/src/plugins/bootstrap/yaml-migrator.ts @@ -18,68 +18,14 @@ import { installNpmPackage, extractPackageManifest, PLUGINS_NODE_MODULES, + resolveCredentials, + resolveOptionalConfig, + findConfigFile, } from '@src/plugins/loader/plugin-fs.js'; import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; import type { YamlConfig } from '@src/types/types.js'; -/** - * Resolve environment variables in credential templates - */ -function resolveCredentials(credentials?: Record): Record { - const resolved: Record = {}; - for (const [k, v] of Object.entries(credentials || {})) { - if (typeof v === 'string' && v.startsWith('${') && v.endsWith('}')) { - const envName = v.slice(2, -1); - resolved[k] = process.env[envName] || ''; - } else { - resolved[k] = v; - } - } - return resolved; -} - -/** - * Resolve optional config placeholders and merge into options - */ -function resolveOptionalConfig(optionalConfig?: Record): Record { - const resolved: Record = {}; - for (const [k, v] of Object.entries(optionalConfig || {})) { - if (typeof v === 'string' && v.startsWith('${') && v.endsWith('}')) { - const envName = v.slice(2, -1); - if (process.env[envName]) { - resolved[k] = process.env[envName]!; - } - } else if (v !== undefined) { - resolved[k] = v; - } - } - return resolved; -} - -/** - * Find local config file if it exists - */ -function findConfigPath(): string | null { - const customPath = process.env.SIMPLENS_CONFIG_PATH; - if (customPath && existsSync(customPath)) { - return customPath; - } - - const candidates = [ - './simplens.config.yaml', - './simplens.config.yml', - './simplens.config.json', - join(process.cwd(), 'simplens.config.yaml'), - join(process.cwd(), 'simplens.config.yml'), - ]; - - for (const p of candidates) { - if (existsSync(p)) return p; - } - return null; -} - export class YamlMigrator { /** * Check if MongoDB has providers; if empty and config file exists, migrate to MongoDB. @@ -94,7 +40,8 @@ export class YamlMigrator { return false; } - const configPath = findConfigPath(); + + const configPath = findConfigFile(); if (!configPath) { logger.info('No simplens.config.yaml found and MongoDB is empty. Standby mode ready.'); return false; diff --git a/src/plugins/index.ts b/src/plugins/index.ts index 4a2f64d..b076987 100644 --- a/src/plugins/index.ts +++ b/src/plugins/index.ts @@ -21,7 +21,6 @@ export { loadProvidersFromEnv, loadProvidersFromDatabase, getConfiguredChannels, - getConfiguredChannelsFromDatabase, registerProvider, } from './loader/loader.js'; diff --git a/src/plugins/interfaces/provider.types.ts b/src/plugins/interfaces/provider.types.ts deleted file mode 100644 index bab2ec2..0000000 --- a/src/plugins/interfaces/provider.types.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** - * Provider contracts are owned by the shared type module. These exports keep - * the existing plugin import path compatible with external providers. - */ -export type { - BaseNotification, - DeliveryResult, - ProviderConfig, - ProviderManifest, - RateLimitConfig, - SimpleNSProvider, -} from '@src/types/types.js'; diff --git a/src/plugins/loader/loader.ts b/src/plugins/loader/loader.ts index 300e6d6..8469837 100644 --- a/src/plugins/loader/loader.ts +++ b/src/plugins/loader/loader.ts @@ -5,46 +5,31 @@ * Supports dynamic import of npm packages. */ -import { readFileSync, existsSync, mkdirSync, writeFileSync } from 'fs'; +import { readFileSync, existsSync } from 'fs'; import { join } from 'path'; -import { execSync } from 'child_process'; -import { pathToFileURL } from 'url'; import { parse as parseYaml } from 'yaml'; import { PluginRegistry, type ChannelConfig } from './registry.js'; -import type { SimpleNSProvider, ProviderConfig } from '../interfaces/provider.types.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; import Plugin from '@src/database/models/plugin.models.js'; import Provider from '@src/database/models/provider.models.js'; import ChannelRouting from '@src/database/models/channel-routing.models.js'; import { ProviderManagerService } from '@src/plugins/services/provider-manager.service.js'; -import { installNpmPackage } from './plugin-fs.js'; -import type { provider_document, ProviderEntry, ProviderOptions, SimpleNSConfig } from '@src/types/types.js'; - -// Plugins directory for user-installed plugins -const PLUGINS_DIR = join(process.cwd(), '.plugins'); -const PLUGINS_NODE_MODULES = join(PLUGINS_DIR, 'node_modules'); - -/** - * Initialize plugins directory with package.json if needed - */ -function initPluginsDir(): void { - if (!existsSync(PLUGINS_DIR)) { - mkdirSync(PLUGINS_DIR, { recursive: true }); - } - - const packageJsonPath = join(PLUGINS_DIR, 'package.json'); - if (!existsSync(packageJsonPath)) { - const initialPackage = { - name: 'simplens-plugins', - version: '1.0.0', - description: 'User-installed SimpleNS plugins', - private: true, - type: 'module', - dependencies: {} - }; - writeFileSync(packageJsonPath, JSON.stringify(initialPackage, null, 2)); - } -} +import { + PLUGINS_NODE_MODULES, + installNpmPackage, + importAndInstantiateProvider, + resolveCredentials, + resolveOptionalConfig, + findConfigFile, +} from './plugin-fs.js'; +import type { + provider_document, + ProviderEntry, + ProviderOptions, + SimpleNSConfig, + SimpleNSProvider, + ProviderConfig, +} from '@src/types/types.js'; /** * Install missing plugins from configuration @@ -69,47 +54,10 @@ async function installMissingPlugins(config: SimpleNSConfig): Promise { } logger.info(`Auto-installing ${missingPackages.length} missing plugin(s)...`); - initPluginsDir(); for (const pkg of missingPackages) { - logger.info(`Installing ${pkg}...`); - try { - execSync(`npm install ${pkg}`, { - cwd: PLUGINS_DIR, - stdio: 'pipe' // Suppress output for cleaner logs - }); - logger.success(`Installed ${pkg}`); - } catch (err) { - logger.error(`Failed to install ${pkg}`, err); - const wrappedErr = new Error(`Failed to auto-install plugin: ${pkg}`); - (wrappedErr as unknown as Record).cause = err; - throw wrappedErr; - } - } -} - -/** - * Provider entry in configuration - */ -/** - * Find local config file if it exists - * Checks for simplens.config.yaml/.yml/.json - */ -function findLocalConfig(basePath: string): string | null { - const dir = basePath.substring(0, basePath.lastIndexOf('/') + 1) || './'; - const localPatterns = [ - 'simplens.config.yaml', - 'simplens.config.yml', - 'simplens.config.json' - ]; - - for (const filename of localPatterns) { - const localPath = dir + filename; - if (existsSync(localPath)) { - return localPath; - } + installNpmPackage(pkg); } - return null; } /** @@ -117,13 +65,7 @@ function findLocalConfig(basePath: string): string | null { * Returns null if no config file exists */ function loadConfig(configPath: string): SimpleNSConfig | null { - // Check for local config first (takes precedence) - const localConfigPath = findLocalConfig(configPath); - const pathToUse = localConfigPath || configPath; - - if (localConfigPath) { - logger.info(`Using local config: ${localConfigPath}`); - } + const pathToUse = findConfigFile(configPath) || configPath; if (!existsSync(pathToUse)) { // No config file - this is okay, just means no plugins configured @@ -142,144 +84,6 @@ function loadConfig(configPath: string): SimpleNSConfig | null { } } -/** - * Resolve environment variables in credentials - * Supports ${VAR_NAME} syntax - */ -function resolveCredentials(credentials: Record | undefined): Record { - const resolved: Record = {}; - - for (const [key, value] of Object.entries(credentials || {})) { - if (typeof value === 'string' && value.startsWith('${') && value.endsWith('}')) { - const envVar = value.slice(2, -1); - const envValue = process.env[envVar]; - if (!envValue) { - logger.warn(`Environment variable ${envVar} not set`); - } - resolved[key] = envValue || ''; - } else { - resolved[key] = value; - } - } - - return resolved; -} - -/** - * Resolve optional config from environment variables - * Takes the optionalConfig from the config entry and resolves ${VAR} placeholders - * If env var is not set, the key is removed (no error since it's optional) - */ -function resolveOptionalConfig( - optionalConfig: Record | undefined -): Record { - const resolved: Record = {}; - - if (!optionalConfig) return resolved; - - for (const [key, value] of Object.entries(optionalConfig)) { - // Check if value is a placeholder like ${VAR_NAME} - if (typeof value === 'string' && value.startsWith('${') && value.endsWith('}')) { - const envVar = value.slice(2, -1); - const envValue = process.env[envVar]; - - if (envValue) { - resolved[key] = envValue; - logger.debug(`Loaded optional config: ${key}`); - } - // If not set, just skip (don't add to resolved) - } else { - // Literal value, keep as-is - resolved[key] = value; - } - } - - return resolved; -} - -/** - * Helper to instantiate provider from module - */ -function instantiateFromModule(module: Record, packageName: string): SimpleNSProvider { - if (module.default) { - if (typeof module.default === 'function') { - const DefaultExport = module.default as new () => SimpleNSProvider; - if (DefaultExport.prototype?.constructor) { - return new DefaultExport(); - } - // Factory function - return (module.default as () => SimpleNSProvider)(); - } - return module.default as SimpleNSProvider; - } - if (module.createProvider) { - return (module.createProvider as () => SimpleNSProvider)(); - } - throw new Error(`Package ${packageName} does not export a provider`); -} - -/** - * Resolve the entry file for a package from its package.json - */ -function resolvePackageEntry(packagePath: string): string { - const pkgJsonPath = join(packagePath, 'package.json'); - if (!existsSync(pkgJsonPath)) { - return join(packagePath, 'index.js'); // Fallback - } - - const pkg = JSON.parse(readFileSync(pkgJsonPath, 'utf-8')); - - // Resolve entry point: exports > main > index.js - let entryPoint = 'index.js'; - if (pkg.exports) { - if (typeof pkg.exports === 'string') { - entryPoint = pkg.exports; - } else if (pkg.exports['.']) { - const dotExport = pkg.exports['.']; - entryPoint = typeof dotExport === 'string' - ? dotExport - : (dotExport.import || dotExport.default || 'index.js'); - } - } else if (pkg.main) { - entryPoint = pkg.main; - } - - return join(packagePath, entryPoint); -} - -/** - * Dynamically import a provider package and instantiate - * First checks plugins directory, then falls back to core node_modules - */ -async function importAndInstantiateProvider(packageName: string): Promise { - try { - // First, try to import from plugins directory - const pluginPath = join(PLUGINS_NODE_MODULES, packageName); - if (existsSync(pluginPath)) { - logger.debug(`Loading from plugins directory: ${packageName}`); - // Resolve the actual entry file from package.json - const entryFile = resolvePackageEntry(pluginPath); - // Convert to file:// URL for cross-platform compatibility - const pluginUrl = pathToFileURL(entryFile).href; - const module = await import(pluginUrl) as Record; - return instantiateFromModule(module, packageName); - } - - // Fall back to core node_modules (for bundled plugins or testing) - const module = await import(packageName) as Record; - return instantiateFromModule(module, packageName); - } catch (_err) { - // Handle local path imports - if (packageName.startsWith('./') || packageName.startsWith('../')) { - const module = await import(packageName) as Record; - return instantiateFromModule(module, packageName); - } - const wrappedErr = new Error(`Plugin not found: ${packageName}. Install with: npm run plugin:install ${packageName}`); - (wrappedErr as unknown as Record).cause = _err; - throw wrappedErr; - } -} - /** * Load a single provider */ @@ -478,19 +282,3 @@ export async function loadProvidersFromDatabase(options: { initialize?: boolean logger.info(`Configured channels: ${PluginRegistry.getChannels().join(', ')}`); } -/** - * Get list of configured channels from MongoDB (falls back to YAML if DB has no routings) - */ -export async function getConfiguredChannelsFromDatabase(): Promise { - try { - const routings = await ChannelRouting.find().lean(); - if (routings && routings.length > 0) { - return routings.map((r) => r.channel); - } - } catch (err) { - logger.warn('Could not query ChannelRouting from database:', { - error: err instanceof Error ? err.message : String(err) - }); - } - return getConfiguredChannels(); -} diff --git a/src/plugins/loader/plugin-fs.ts b/src/plugins/loader/plugin-fs.ts index ba52a3c..f2029f6 100644 --- a/src/plugins/loader/plugin-fs.ts +++ b/src/plugins/loader/plugin-fs.ts @@ -9,12 +9,84 @@ import { readFileSync, existsSync, mkdirSync, writeFileSync } from 'fs'; import { join } from 'path'; import { execSync } from 'child_process'; import { pathToFileURL } from 'url'; -import type { SimpleNSProvider, ProviderManifest } from '../interfaces/provider.types.js'; +import type { SimpleNSProvider, ProviderManifest } from '@src/types/types.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; export const PLUGINS_DIR = join(process.cwd(), '.plugins'); export const PLUGINS_NODE_MODULES = join(PLUGINS_DIR, 'node_modules'); +/** + * Resolve environment variables in credentials (${VAR_NAME} syntax) + */ +export function resolveCredentials(credentials: Record | undefined): Record { + const resolved: Record = {}; + + for (const [key, value] of Object.entries(credentials || {})) { + if (typeof value === 'string' && value.startsWith('${') && value.endsWith('}')) { + const envVar = value.slice(2, -1); + const envValue = process.env[envVar]; + if (!envValue) { + logger.warn(`Environment variable ${envVar} not set`); + } + resolved[key] = envValue || ''; + } else { + resolved[key] = value; + } + } + + return resolved; +} + +/** + * Resolve optional config from environment variables (${VAR_NAME} syntax) + */ +export function resolveOptionalConfig( + optionalConfig: Record | undefined +): Record { + const resolved: Record = {}; + if (!optionalConfig) return resolved; + + for (const [key, value] of Object.entries(optionalConfig)) { + if (typeof value === 'string' && value.startsWith('${') && value.endsWith('}')) { + const envVar = value.slice(2, -1); + const envValue = process.env[envVar]; + if (envValue) { + resolved[key] = envValue; + logger.debug(`Loaded optional config: ${key}`); + } + } else { + resolved[key] = value; + } + } + + return resolved; +} + +/** + * Find local config file if it exists + */ +export function findConfigFile(basePath?: string): string | null { + const customPath = process.env.SIMPLENS_CONFIG_PATH; + if (customPath && existsSync(customPath)) { + return customPath; + } + + const dir = basePath ? basePath.substring(0, basePath.lastIndexOf('/') + 1) || './' : './'; + const candidates = [ + dir + 'simplens.config.yaml', + dir + 'simplens.config.yml', + dir + 'simplens.config.json', + join(process.cwd(), 'simplens.config.yaml'), + join(process.cwd(), 'simplens.config.yml'), + join(process.cwd(), 'simplens.config.json'), + ]; + + for (const p of candidates) { + if (existsSync(p)) return p; + } + return null; +} + /** * Initialize plugins directory with package.json if needed */ diff --git a/src/plugins/loader/registry.ts b/src/plugins/loader/registry.ts index 89be81b..296793b 100644 --- a/src/plugins/loader/registry.ts +++ b/src/plugins/loader/registry.ts @@ -256,14 +256,6 @@ class PluginRegistryClass { return providers[0]; } - /** - * Get first fallback provider ID for a channel - */ - getFallbackProviderId(channel: string): string | undefined { - const ids = this.getFallbackProviderIds(channel); - return ids[0]; - } - /** * Get all registered provider IDs */ diff --git a/src/plugins/loader/router.ts b/src/plugins/loader/router.ts index e09b8f5..cecb6d6 100644 --- a/src/plugins/loader/router.ts +++ b/src/plugins/loader/router.ts @@ -5,7 +5,7 @@ */ import { PluginRegistry } from './registry.js'; -import type { DeliveryResult, BaseNotification } from '../interfaces/provider.types.js'; +import type { DeliveryResult, BaseNotification } from '@src/types/types.js'; import { unifiedProcessorLogger as logger } from '@src/processors/unified/unified.logger.js'; import { handleSchemaValidationFailure } from '@src/processors/shared/schema-failure-handler.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; @@ -265,23 +265,13 @@ export function validateNotification( channel: string, notification: unknown ): { success: true; data: unknown } | { success: false; error: string } { - const provider = PluginRegistry.getDefaultProvider(channel); + const providerId = PluginRegistry.getDefaultProviderId(channel); - if (!provider) { + if (!providerId) { return { success: false, error: `No provider for channel: ${channel}` }; } - const schema = provider.getNotificationSchema(); - const result = schema.safeParse(notification); - - if (result.success) { - return { success: true, data: result.data }; - } - - return { - success: false, - error: result.error.issues.map(i => `${i.path.join('.')}: ${i.message}`).join(', '), - }; + return validateNotificationForProvider(providerId, notification); } /** diff --git a/src/plugins/services/plugin-manager.service.ts b/src/plugins/services/plugin-manager.service.ts index 999864f..bf9cfc4 100644 --- a/src/plugins/services/plugin-manager.service.ts +++ b/src/plugins/services/plugin-manager.service.ts @@ -21,14 +21,14 @@ import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; export class PluginManagerService { /** - * Install an npm package, extract its manifest, persist to MongoDB, and broadcast. + * Helper to install package, extract manifest, upsert/update Plugin record, and broadcast sync event */ - public static async installPlugin( + private static async saveAndBroadcastPlugin( packageName: string, - version?: string + version: string | undefined, + syncAction: 'PLUGIN_INSTALLED' | 'PLUGIN_VERSION_CHANGED', + upsert = false ): Promise { - logger.info(`Installing plugin package '${packageName}' (version: ${version || 'latest'})...`); - return await PluginSyncService.runWithLock(async () => { try { installNpmPackage(packageName, version); @@ -43,23 +43,36 @@ export class PluginManagerService { manifest, error: undefined, }, - { upsert: true, new: true } + upsert ? { upsert: true, new: true } : { new: true } ); - await PluginSyncService.publish('PLUGIN_INSTALLED', { + await PluginSyncService.publish(syncAction, { plugin_name: packageName, version: authoritativeVersion, }); - logger.success(`Plugin '${packageName}'@${authoritativeVersion} installed successfully.`); - return plugin.toObject() as plugin_document; + logger.success( + `Plugin '${packageName}' ${upsert ? '@' + authoritativeVersion + ' installed' : 'updated to version ' + authoritativeVersion} successfully.` + ); + return plugin!.toObject() as plugin_document; } catch (err) { - logger.error(`Failed to install plugin '${packageName}':`, err); + logger.error(`Failed to process plugin '${packageName}':`, err); throw err; } }); } + /** + * Install an npm package, extract its manifest, persist to MongoDB, and broadcast. + */ + public static async installPlugin( + packageName: string, + version?: string + ): Promise { + logger.info(`Installing plugin package '${packageName}' (version: ${version || 'latest'})...`); + return this.saveAndBroadcastPlugin(packageName, version, 'PLUGIN_INSTALLED', true); + } + /** * Change/upgrade/downgrade the version of an already installed plugin. */ @@ -74,34 +87,7 @@ export class PluginManagerService { throw new Error(`Plugin '${packageName}' is not installed.`); } - return await PluginSyncService.runWithLock(async () => { - try { - installNpmPackage(packageName, version); - const { manifest, version: authoritativeVersion } = await extractPackageManifest(packageName); - - const plugin = await Plugin.findOneAndUpdate( - { name: packageName }, - { - version: authoritativeVersion, - status: 'installed', - manifest, - error: undefined, - }, - { new: true } - ); - - await PluginSyncService.publish('PLUGIN_VERSION_CHANGED', { - plugin_name: packageName, - version: authoritativeVersion, - }); - - logger.success(`Plugin '${packageName}' updated to version ${authoritativeVersion}.`); - return plugin!.toObject() as plugin_document; - } catch (err) { - logger.error(`Failed to update plugin '${packageName}' to version '${version}':`, err); - throw err; - } - }); + return this.saveAndBroadcastPlugin(packageName, version, 'PLUGIN_VERSION_CHANGED', false); } /** @@ -181,30 +167,25 @@ export class PluginManagerService { * Register remote synchronization handlers with Redis Pub/Sub */ public static registerSyncHandlers(): void { - PluginSyncService.on('PLUGIN_INSTALLED', async (msg) => { - const { plugin_name, version } = msg.payload; + const handleRemoteInstallOrUpdate = async (actionDesc: string, payload: { plugin_name?: string; version?: string }) => { + const { plugin_name, version } = payload; if (!plugin_name) return; - logger.info(`Sync: Installing remote plugin '${plugin_name}' (${version || 'latest'})...`); + logger.info(`Sync: ${actionDesc} for '${plugin_name}' (${version || 'latest'})...`); await PluginSyncService.runWithLock(async () => { try { installNpmPackage(plugin_name, version); } catch (err) { - logger.error(`Failed to sync remote install of '${plugin_name}':`, err); + logger.error(`Failed to sync remote ${actionDesc} for '${plugin_name}':`, err); } }); + }; + + PluginSyncService.on('PLUGIN_INSTALLED', async (msg) => { + await handleRemoteInstallOrUpdate('Installing remote plugin', msg.payload); }); PluginSyncService.on('PLUGIN_VERSION_CHANGED', async (msg) => { - const { plugin_name, version } = msg.payload; - if (!plugin_name) return; - logger.info(`Sync: Changing remote plugin '${plugin_name}' to version '${version}'...`); - await PluginSyncService.runWithLock(async () => { - try { - installNpmPackage(plugin_name, version); - } catch (err) { - logger.error(`Failed to sync remote version change of '${plugin_name}':`, err); - } - }); + await handleRemoteInstallOrUpdate('Changing remote plugin version', msg.payload); }); PluginSyncService.on('PLUGIN_UNINSTALLED', async (msg) => { diff --git a/src/plugins/services/provider-manager.service.ts b/src/plugins/services/provider-manager.service.ts index a2d2abb..f9c4562 100644 --- a/src/plugins/services/provider-manager.service.ts +++ b/src/plugins/services/provider-manager.service.ts @@ -15,7 +15,14 @@ import { import { importAndInstantiateProvider } from '@src/plugins/loader/plugin-fs.js'; import { PluginRegistry } from '@src/plugins/loader/registry.js'; import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; -import type { provider_document, ProviderConfig, ProviderOptions, ProviderResponseDto } from '@src/types/types.js'; +import type { + provider_document, + ProviderConfig, + ProviderOptions, + ProviderResponseDto, + SimpleNSProvider, + encrypted_credentials, +} from '@src/types/types.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; export type { ProviderResponseDto } from '@src/types/types.js'; @@ -286,6 +293,47 @@ export class ProviderManagerService { }; } + /** + * Helper to retrieve cached credentials or decrypt and cache them + */ + private static async getOrDecryptCredentials( + id: string, + encryptedCredentials?: Record | null + ): Promise> { + let credentials = ProviderManagerService.getCachedCredentials(id); + if (!credentials && encryptedCredentials) { + credentials = await decryptCredentials(encryptedCredentials as unknown as encrypted_credentials); + ProviderManagerService.cacheCredentials(id, credentials); + } + return credentials || {}; + } + + /** + * Helper to instantiate and optionally initialize a provider instance + */ + private static async createAndInitProvider( + id: string, + pluginName: string, + credentials: Record, + options: ProviderOptions, + initialize = true + ): Promise<{ providerInstance: SimpleNSProvider; healthy: boolean }> { + const providerInstance = await importAndInstantiateProvider(pluginName); + let healthy = true; + + if (initialize) { + const config: ProviderConfig = { + id, + credentials, + options, + }; + await providerInstance.initialize(config); + healthy = await providerInstance.healthCheck(); + } + + return { providerInstance, healthy }; + } + /** * Test live provider connection / credentials */ @@ -307,11 +355,7 @@ export class ProviderManagerService { pluginName = existing.plugin_name; options = { ...(existing.options as ProviderOptions), ...options }; if (!credentials || Object.keys(credentials).length === 0) { - credentials = ProviderManagerService.getCachedCredentials(data.provider_id); - if (!credentials) { - credentials = await decryptCredentials(existing.credentials); - ProviderManagerService.cacheCredentials(data.provider_id, credentials); - } + credentials = await ProviderManagerService.getOrDecryptCredentials(data.provider_id, existing.credentials); } } @@ -323,15 +367,13 @@ export class ProviderManagerService { } try { - const providerInstance = await importAndInstantiateProvider(pluginName); - const config: ProviderConfig = { - id: data.provider_id || 'test-connection', + const { healthy } = await ProviderManagerService.createAndInitProvider( + data.provider_id || 'test-connection', + pluginName, credentials, options, - }; - - await providerInstance.initialize(config); - const healthy = await providerInstance.healthCheck(); + true + ); if (healthy) { return { success: true, message: 'Provider connection and health check succeeded.' }; @@ -351,28 +393,17 @@ export class ProviderManagerService { providerDoc: provider_document, initialize = true ): Promise { - let credentials = ProviderManagerService.getCachedCredentials(providerDoc.id); - if (!credentials) { - credentials = await decryptCredentials(providerDoc.credentials); - ProviderManagerService.cacheCredentials(providerDoc.id, credentials); - } - const providerInstance = await importAndInstantiateProvider(providerDoc.plugin_name); - - if (initialize) { - const config: ProviderConfig = { - id: providerDoc.id, - credentials, - options: { - ...(providerDoc.options as ProviderOptions), - }, - }; - - await providerInstance.initialize(config); + const credentials = await ProviderManagerService.getOrDecryptCredentials(providerDoc.id, providerDoc.credentials); + const { providerInstance, healthy } = await ProviderManagerService.createAndInitProvider( + providerDoc.id, + providerDoc.plugin_name, + credentials, + (providerDoc.options as ProviderOptions) || {}, + initialize + ); - const healthy = await providerInstance.healthCheck(); - if (!healthy) { - logger.warn(`Provider '${providerDoc.id}' failed health check on initial load.`); - } + if (initialize && !healthy) { + logger.warn(`Provider '${providerDoc.id}' failed health check on initial load.`); } PluginRegistry.registerOrReplace( diff --git a/src/plugins/sync/plugin-sync.service.ts b/src/plugins/sync/plugin-sync.service.ts index 44f76dc..97f5ae3 100644 --- a/src/plugins/sync/plugin-sync.service.ts +++ b/src/plugins/sync/plugin-sync.service.ts @@ -32,10 +32,6 @@ export class PluginSyncServiceClass { return PluginSyncServiceClass.instance; } - public getInstanceId(): string { - return this.instanceId; - } - /** * Run an asynchronous task with in-process mutex locking * Useful for protecting the .plugins/ folder during npm install/uninstall diff --git a/src/processors/unified/unified.consumer.ts b/src/processors/unified/unified.consumer.ts index b96cdcb..643f181 100644 --- a/src/processors/unified/unified.consumer.ts +++ b/src/processors/unified/unified.consumer.ts @@ -23,7 +23,7 @@ import { resolveFallbackProviderId, validateNotificationForProvider } from '@src/plugins/index.js'; -import type { BaseNotification, DeliveryResult } from '@src/plugins/interfaces/provider.types.js'; +import type { BaseNotification, DeliveryResult } from '@src/types/types.js'; // Shared utilities import { tryAcquireProcessingLock, setDelivered, setFailed, setRateLimited } from '@src/processors/shared/idempotency.js'; From 2fb2b90a5c64b7dfa3144f6000ca617cc6d4bf91 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Wed, 16 Sep 2026 19:40:11 +0530 Subject: [PATCH 13/46] feat: implement dynamic Kafka topic management and ensure topic creation for channel routing --- docker-compose/docker-compose.local.yaml | 4 +- src/config/kafka.config.ts | 33 +++- .../services/channel-routing.service.ts | 28 +-- .../services/provider-manager.service.ts | 18 ++ src/processors/unified/unified.consumer.ts | 5 +- .../unit/config/kafka-dynamic-topics.test.ts | 173 ++++++++++++++++++ .../plugins/channel-routing.service.test.ts | 29 +++ .../plugins/provider-manager.service.test.ts | 51 +++++- .../unit/processors/unified.consumer.test.ts | 1 + 9 files changed, 323 insertions(+), 19 deletions(-) create mode 100644 tests/unit/config/kafka-dynamic-topics.test.ts diff --git a/docker-compose/docker-compose.local.yaml b/docker-compose/docker-compose.local.yaml index f61d676..f8732e6 100644 --- a/docker-compose/docker-compose.local.yaml +++ b/docker-compose/docker-compose.local.yaml @@ -30,7 +30,7 @@ services: SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} volumes: - logs-data-local:/app/logs - - ../simplens.config.yaml:/app/simplens.config.yaml:ro + # - ../simplens.config.yaml:/app/simplens.config.yaml:ro command: [ "node", "dist/api/server.js" ] networks: - simplens-infra @@ -90,7 +90,7 @@ services: LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-local:/app/logs - - ../simplens.config.yaml:/app/simplens.config.yaml:ro + # - ../simplens.config.yaml:/app/simplens.config.yaml:ro command: [ "node", "dist/processors/unified/unified.processor.js" ] depends_on: api: diff --git a/src/config/kafka.config.ts b/src/config/kafka.config.ts index 9b4c313..c8fd0c5 100644 --- a/src/config/kafka.config.ts +++ b/src/config/kafka.config.ts @@ -165,7 +165,24 @@ export const createTopics = async (topics: ITopicConfig[]) => { }; /** - * Dynamically expand partition count for a channel topic + * Ensure a channel topic exists in Kafka with at least desiredPartitions + */ +export const ensureChannelTopic = async ( + channel: string, + desiredPartitions: number = 6 +): Promise => { + const topic = getTopicForChannel(channel); + await createTopics([ + { + topic, + numPartitions: desiredPartitions, + replicationFactor: 1, + } + ]); +}; + +/** + * Dynamically expand partition count for a channel topic (or create if missing) */ export const expandTopicPartitions = async ( channel: string, @@ -180,7 +197,18 @@ export const expandTopicPartitions = async ( const topicMetadata = metadata.topics.find(t => t.name === topic); if (!topicMetadata) { - throw new Error(`Topic '${topic}' for channel '${channel}' not found in Kafka`); + logger.info(`Topic '${topic}' for channel '${channel}' not found in Kafka, creating with ${desiredPartitions} partitions...`); + await admin.createTopics({ + topics: [{ + topic, + numPartitions: desiredPartitions, + replicationFactor: 1, + }], + validateOnly: false, + timeout: 30000, + }); + logger.success(`Created Kafka topic '${topic}' with ${desiredPartitions} partitions`); + return { previous: 0, current: desiredPartitions }; } const currentPartitions = topicMetadata.partitions.length; @@ -209,3 +237,4 @@ export const expandTopicPartitions = async ( } }; + diff --git a/src/plugins/services/channel-routing.service.ts b/src/plugins/services/channel-routing.service.ts index 4a61e52..6873f9f 100644 --- a/src/plugins/services/channel-routing.service.ts +++ b/src/plugins/services/channel-routing.service.ts @@ -9,7 +9,7 @@ import ChannelRouting from '@src/database/models/channel-routing.models.js'; import Provider from '@src/database/models/provider.models.js'; import { PluginRegistry } from '@src/plugins/loader/registry.js'; import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; -import { expandTopicPartitions } from '@src/config/kafka.config.js'; +import { expandTopicPartitions, ensureChannelTopic } from '@src/config/kafka.config.js'; import type { channel_routing_document } from '@src/types/types.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; @@ -66,29 +66,31 @@ export class ChannelRoutingService { } } - // 3. Handle dynamic Kafka partition scaling + // 3. Handle dynamic Kafka topic creation and partition scaling const existing = await ChannelRouting.findOne({ channel }); const currentPartitions = existing?.partitions || 6; const desiredPartitions = data.partitions !== undefined ? data.partitions : currentPartitions; - if (desiredPartitions < currentPartitions) { + if (existing && desiredPartitions < currentPartitions) { throw new Error( `Cannot decrease partitions for channel '${channel}' from ${currentPartitions} to ${desiredPartitions}. Kafka partitions can only be increased.` ); } - if (desiredPartitions > currentPartitions) { - logger.info( - `Requesting Kafka partition expansion for channel '${channel}' from ${currentPartitions} to ${desiredPartitions}...` - ); - try { + try { + if (desiredPartitions > currentPartitions) { + logger.info( + `Requesting Kafka partition expansion for channel '${channel}' from ${currentPartitions} to ${desiredPartitions}...` + ); await expandTopicPartitions(channel, desiredPartitions); - } catch (kafkaErr) { - logger.error(`Failed to expand Kafka topic partitions for channel '${channel}':`, kafkaErr); - throw new Error(`Failed to expand Kafka partitions: ${(kafkaErr as Error).message}`, { - cause: kafkaErr, - }); + } else { + await ensureChannelTopic(channel, desiredPartitions); } + } catch (kafkaErr) { + logger.error(`Failed to configure Kafka topic for channel '${channel}':`, kafkaErr); + throw new Error(`Failed to expand Kafka partitions: ${(kafkaErr as Error).message}`, { + cause: kafkaErr, + }); } // 4. Save to MongoDB diff --git a/src/plugins/services/provider-manager.service.ts b/src/plugins/services/provider-manager.service.ts index f9c4562..18210e0 100644 --- a/src/plugins/services/provider-manager.service.ts +++ b/src/plugins/services/provider-manager.service.ts @@ -8,6 +8,7 @@ import Provider from '@src/database/models/provider.models.js'; import Plugin from '@src/database/models/plugin.models.js'; import ChannelRouting from '@src/database/models/channel-routing.models.js'; +import { ChannelRoutingService } from '@src/plugins/services/channel-routing.service.js'; import { encryptCredentials, decryptCredentials, @@ -102,6 +103,23 @@ export class ProviderManagerService { error: err instanceof Error ? err.message : String(err), }); } + + // Automatically create ChannelRouting if none exists for this channel yet + try { + const existingRouting = await ChannelRouting.findOne({ channel: providerDoc.channel }); + if (!existingRouting) { + logger.info(`No channel routing found for '${providerDoc.channel}'. Automatically creating default routing with provider '${data.id}'...`); + await ChannelRoutingService.setChannelRouting(providerDoc.channel, { + default_provider_id: data.id, + fallback_provider_ids: [], + partitions: 6, + }); + } + } catch (routingErr) { + logger.warn(`Failed to auto-create channel routing for '${providerDoc.channel}':`, { + error: routingErr instanceof Error ? routingErr.message : String(routingErr), + }); + } } await PluginSyncService.publish('PROVIDER_UPSERTED', { diff --git a/src/processors/unified/unified.consumer.ts b/src/processors/unified/unified.consumer.ts index 643f181..7fd5187 100644 --- a/src/processors/unified/unified.consumer.ts +++ b/src/processors/unified/unified.consumer.ts @@ -11,7 +11,7 @@ */ import { Consumer, EachMessagePayload } from 'kafkajs'; -import { kafka } from '@src/config/kafka.config.js'; +import { kafka, ensureChannelTopic } from '@src/config/kafka.config.js'; import { NOTIFICATION_STATUS_SF } from '@src/types/types.js'; import { env } from '@src/config/env.config.js'; import { unifiedProcessorLogger as logger } from './unified.logger.js'; @@ -448,6 +448,9 @@ export const startUnifiedConsumer = async (channel: string): Promise => { return; } + // Ensure the Kafka topic exists before attempting to subscribe + await ensureChannelTopic(channel); + const topic = getTopicForChannel(channel); const groupId = getConsumerGroupId(channel); diff --git a/tests/unit/config/kafka-dynamic-topics.test.ts b/tests/unit/config/kafka-dynamic-topics.test.ts new file mode 100644 index 0000000..f47327c --- /dev/null +++ b/tests/unit/config/kafka-dynamic-topics.test.ts @@ -0,0 +1,173 @@ +/** + * Unit Tests for Dynamic Kafka Topic Management in kafka.config.ts + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { kafka, ensureChannelTopic, expandTopicPartitions } from '../../../src/config/kafka.config.js'; + +describe('Dynamic Kafka Topics', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + describe('ensureChannelTopic', () => { + it('should create missing channel topic with default 6 partitions', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + listTopics: vi.fn().mockResolvedValue([]), + createTopics: vi.fn().mockResolvedValue(true), + fetchTopicMetadata: vi.fn().mockResolvedValue({ topics: [] }), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + await ensureChannelTopic('email'); + + expect(adminMock.connect).toHaveBeenCalled(); + expect(adminMock.listTopics).toHaveBeenCalled(); + expect(adminMock.createTopics).toHaveBeenCalledWith({ + topics: [ + { + topic: 'email_notification', + numPartitions: 6, + replicationFactor: 1, + }, + ], + validateOnly: false, + timeout: 30000, + }); + expect(adminMock.disconnect).toHaveBeenCalled(); + }); + + it('should pass custom partition count to createTopics', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + listTopics: vi.fn().mockResolvedValue([]), + createTopics: vi.fn().mockResolvedValue(true), + fetchTopicMetadata: vi.fn().mockResolvedValue({ topics: [] }), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + await ensureChannelTopic('sms', 12); + + expect(adminMock.createTopics).toHaveBeenCalledWith({ + topics: [ + { + topic: 'sms_notification', + numPartitions: 12, + replicationFactor: 1, + }, + ], + validateOnly: false, + timeout: 30000, + }); + }); + + it('should not call admin.createTopics if topic already exists and partitions match', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + listTopics: vi.fn().mockResolvedValue(['email_notification']), + createTopics: vi.fn().mockResolvedValue(true), + fetchTopicMetadata: vi.fn().mockResolvedValue({ + topics: [ + { + name: 'email_notification', + partitions: new Array(6).fill({}), + }, + ], + }), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + await ensureChannelTopic('email', 6); + + expect(adminMock.createTopics).not.toHaveBeenCalled(); + }); + }); + + describe('expandTopicPartitions', () => { + it('should create topic if topic is not found in Kafka metadata', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + fetchTopicMetadata: vi.fn().mockResolvedValue({ + topics: [], + }), + createTopics: vi.fn().mockResolvedValue(true), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + const result = await expandTopicPartitions('push', 8); + + expect(adminMock.createTopics).toHaveBeenCalledWith({ + topics: [ + { + topic: 'push_notification', + numPartitions: 8, + replicationFactor: 1, + }, + ], + validateOnly: false, + timeout: 30000, + }); + expect(result).toEqual({ previous: 0, current: 8 }); + }); + + it('should expand partitions if requested count exceeds current count', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + fetchTopicMetadata: vi.fn().mockResolvedValue({ + topics: [ + { + name: 'email_notification', + partitions: new Array(6).fill({}), + }, + ], + }), + createPartitions: vi.fn().mockResolvedValue(true), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + const result = await expandTopicPartitions('email', 10); + + expect(adminMock.createPartitions).toHaveBeenCalledWith({ + topicPartitions: [ + { + topic: 'email_notification', + count: 10, + }, + ], + }); + expect(result).toEqual({ previous: 6, current: 10 }); + }); + + it('should throw error if attempting to decrease partitions', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + fetchTopicMetadata: vi.fn().mockResolvedValue({ + topics: [ + { + name: 'email_notification', + partitions: new Array(10).fill({}), + }, + ], + }), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + await expect(expandTopicPartitions('email', 4)).rejects.toThrow( + 'Kafka partition count cannot be decreased' + ); + }); + }); +}); diff --git a/tests/unit/plugins/channel-routing.service.test.ts b/tests/unit/plugins/channel-routing.service.test.ts index bcf4f99..ba02e1e 100644 --- a/tests/unit/plugins/channel-routing.service.test.ts +++ b/tests/unit/plugins/channel-routing.service.test.ts @@ -110,5 +110,34 @@ describe('ChannelRoutingService', () => { }); expect(result.partitions).toBe(12); }); + + it('should ensure Kafka topic is created when configuring new channel routing', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ + id: 'email-primary', + channel: 'email', + } as any); + + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue(null); + + const ensureTopicSpy = vi + .spyOn(kafkaConfig, 'ensureChannelTopic') + .mockResolvedValue(undefined); + + const updatedRouting = { + channel: 'email', + default_provider_id: 'email-primary', + fallback_provider_ids: [], + partitions: 6, + toObject: () => updatedRouting, + }; + + vi.spyOn(ChannelRouting, 'findOneAndUpdate').mockResolvedValue(updatedRouting as any); + + await ChannelRoutingService.setChannelRouting('email', { + default_provider_id: 'email-primary', + }); + + expect(ensureTopicSpy).toHaveBeenCalledWith('email', 6); + }); }); }); diff --git a/tests/unit/plugins/provider-manager.service.test.ts b/tests/unit/plugins/provider-manager.service.test.ts index f9c9320..44e3d03 100644 --- a/tests/unit/plugins/provider-manager.service.test.ts +++ b/tests/unit/plugins/provider-manager.service.test.ts @@ -11,6 +11,7 @@ import * as keypairManager from '../../../src/plugins/crypto/keypair-manager.js' import * as pluginFs from '../../../src/plugins/loader/plugin-fs.js'; import { PluginRegistry } from '../../../src/plugins/loader/registry.js'; import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; +import { ChannelRoutingService } from '../../../src/plugins/services/channel-routing.service.js'; describe('ProviderManagerService', () => { beforeEach(() => { @@ -71,7 +72,8 @@ describe('ProviderManagerService', () => { }; vi.spyOn(Provider, 'create').mockResolvedValue(mockCreated as any); - vi.spyOn(ProviderManagerService, 'loadAndRegisterProvider').mockResolvedValue(undefined); + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue(null); + const setRoutingSpy = vi.spyOn(ChannelRoutingService, 'setChannelRouting').mockResolvedValue({} as any); const result = await ProviderManagerService.createProvider({ id: 'mock-test', @@ -89,11 +91,58 @@ describe('ProviderManagerService', () => { options: { priority: 5 }, }) ); + expect(setRoutingSpy).toHaveBeenCalledWith('mock', { + default_provider_id: 'mock-test', + fallback_provider_ids: [], + partitions: 6, + }); expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_UPSERTED', { provider_id: 'mock-test', }); expect(result.credentials_configured).toBe(true); }); + + it('should not overwrite ChannelRouting if one already exists for the channel', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue(null); + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ + name: '@simplens/mock', + manifest: { channel: 'mock' }, + } as any); + + vi.spyOn(keypairManager, 'encryptCredentials').mockResolvedValue({ + encrypted_data: 'enc123', + iv: 'iv123', + auth_tag: 'tag123', + encrypted_dek: 'dek123', + }); + + const mockCreated = { + _id: '507f1f77bcf86cd799439011', + id: 'mock-test-2', + plugin_name: '@simplens/mock', + channel: 'mock', + enabled: true, + credentials: { encrypted_data: 'enc123' }, + options: {}, + toObject: () => mockCreated, + }; + + vi.spyOn(Provider, 'create').mockResolvedValue(mockCreated as any); + vi.spyOn(ProviderManagerService, 'loadAndRegisterProvider').mockResolvedValue(undefined); + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue({ + channel: 'mock', + default_provider_id: 'existing-provider', + } as any); + const setRoutingSpy = vi.spyOn(ChannelRoutingService, 'setChannelRouting'); + + await ProviderManagerService.createProvider({ + id: 'mock-test-2', + plugin_name: '@simplens/mock', + credentials: {}, + }); + + expect(setRoutingSpy).not.toHaveBeenCalled(); + }); }); describe('deleteProvider', () => { diff --git a/tests/unit/processors/unified.consumer.test.ts b/tests/unit/processors/unified.consumer.test.ts index 77d591b..571dc42 100644 --- a/tests/unit/processors/unified.consumer.test.ts +++ b/tests/unit/processors/unified.consumer.test.ts @@ -42,6 +42,7 @@ vi.mock('../../../src/config/kafka.config.js', () => ({ kafka: { consumer: vi.fn(() => kafkaConsumerMock), }, + ensureChannelTopic: vi.fn().mockResolvedValue(undefined), })); vi.mock('../../../src/plugins/index.js', () => ({ From 3be71eb65874249df8c4fdb462969520dad6fac6 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Fri, 18 Sep 2026 11:46:38 +0530 Subject: [PATCH 14/46] feat: Implement dynamic configuration for operational settings and admin authentication - Added dynamic configuration service to manage operational settings with zero-restart updates. - Refactored environment variable handling to utilize dynamic settings for various worker parameters. - Updated system configuration model to support mixed types for dynamic settings. - Enhanced delayed and recovery processors to listen for dynamic configuration changes and adjust intervals accordingly. - Created integration tests for admin authentication routes and settings API. - Added unit tests for dynamic configuration schema validations and functionality. --- .env.example | 123 +--- dashboard/.env.example | 4 +- dashboard/app/api/auth/login/route.ts | 40 +- dashboard/app/api/auth/setup/route.ts | 70 ++ dashboard/app/api/auth/status/route.ts | 21 + dashboard/app/login/page.tsx | 20 +- dashboard/app/settings/page.tsx | 695 ++++++++++++++++++- dashboard/app/setup/page.tsx | 381 ++++++++++ dashboard/lib/api-client.ts | 15 + dashboard/lib/session.ts | 12 +- dashboard/lib/types.ts | 41 ++ dashboard/middleware.ts | 11 +- src/api/controllers/admin-auth.controller.ts | 152 ++++ src/api/controllers/settings.controller.ts | 76 ++ src/api/routes/admin-auth.routes.ts | 19 + src/api/routes/settings.routes.ts | 19 + src/api/server.ts | 8 + src/config/dynamic-config.service.ts | 264 +++++++ src/config/env.config.ts | 117 +++- src/database/models/system-config.models.ts | 4 +- src/processors/delayed/delayed.poller.ts | 10 + src/processors/delayed/delayed.processor.ts | 17 +- src/processors/unified/unified.processor.ts | 4 + src/types/schemas.ts | 93 ++- src/types/types.ts | 40 +- src/workers/consumers/status.consumer.ts | 2 +- src/workers/cron/background.cron.ts | 28 +- src/workers/recovery/recovery.cron.ts | 17 +- src/workers/recovery/recovery.service.ts | 5 + src/workers/utils/logger.ts | 40 +- src/workers/worker.ts | 4 + tests/integration/api/admin-auth.test.ts | 108 +++ tests/integration/api/settings.test.ts | 97 +++ tests/unit/config/dynamic-config.test.ts | 135 ++++ 34 files changed, 2475 insertions(+), 217 deletions(-) create mode 100644 dashboard/app/api/auth/setup/route.ts create mode 100644 dashboard/app/api/auth/status/route.ts create mode 100644 dashboard/app/setup/page.tsx create mode 100644 src/api/controllers/admin-auth.controller.ts create mode 100644 src/api/controllers/settings.controller.ts create mode 100644 src/api/routes/admin-auth.routes.ts create mode 100644 src/api/routes/settings.routes.ts create mode 100644 src/config/dynamic-config.service.ts create mode 100644 tests/integration/api/admin-auth.test.ts create mode 100644 tests/integration/api/settings.test.ts create mode 100644 tests/unit/config/dynamic-config.test.ts diff --git a/.env.example b/.env.example index 85f85e4..a9d680b 100644 --- a/.env.example +++ b/.env.example @@ -1,122 +1,53 @@ -# ============================================ -# INFRASTRUCTURE HOST CONFIGURATION -# ============================================ -# INFRA_HOST: Used by docker-compose.infra.yaml for Kafka & MongoDB configuration -# - Same system: Leave unset (defaults to host.docker.internal) -# - Separate systems: Set to the infrastructure machine's IP/hostname -# ============================================ -# INFRA_HOST=192.168.1.100 # Uncomment and set when running infra on a separate machine +# ============================================================================== +# SimpleNS Essential Environment Configuration +# ============================================================================== +# NOTE: All operational settings (worker batch sizes, poll intervals, retries, +# deduplication TTLs, recovery thresholds, and log levels) are now managed +# dynamically via MongoDB and the Admin Dashboard with live zero-downtime hot-reloading! +# +# Administrator credentials are configured interactively on first launch via +# the Dashboard Setup UI (/setup) and stored securely in MongoDB. +# ============================================================================== -# ============================================ -# CONNECTION URLS (for application services in docker-compose.yaml) -# ============================================ -# Same system: Use host.docker.internal as the host -# Separate systems: Use the infrastructure machine's IP/hostname -# ============================================ - -NODE_ENV= # Set to either production or development # ============================================ # API SERVER # ============================================ -NS_API_KEY= # Generate via: openssl rand -base64 32 +NODE_ENV=production # Set to 'production', 'development', or 'test' +# Generate via: openssl rand -base64 32 +NS_API_KEY= # API secret key for authenticating API requests PORT=3000 # API server port -MAX_BATCH_REQ_LIMIT=1000 # Max batch request size # ============================================ # DATABASE # ============================================ -MONGO_URI= # MongoDB connection string (If you are using self hosted instance, it should be configured with replicaSet: mongodb://host:port/simplens?replicaSet=rs0) +# MongoDB connection URI (configured with replicaSet for change streams/transactions) +MONGO_URI=mongodb://127.0.0.1:27017/notification_service?replicaSet=rs0 # ============================================ # KAFKA # ============================================ -BROKERS= # Kafka brokers list (host:port if self-hosted) - -# Kafka Topic Partitions (Core Topics) -DELAYED_PARTITION=1 # Partitions for delayed_notification topic -NOTIFICATION_STATUS_PARTITION=1 # Partitions for notification_status topic - +BROKERS=localhost:9092 # Comma-separated Kafka broker addresses # ============================================ # REDIS # ============================================ -REDIS_URL= # Redis connection URL (redis://host:port if self-hosted) - -# ============================================ -# PLUGIN SYSTEM -# ============================================ -SIMPLENS_CONFIG_PATH=./simplens.config.yaml # Path to plugin configuration file -PROCESSOR_CHANNEL=all # 'all' for all channels, or specific: 'email', 'sms' - - -# ============================================ -# BACKGROUND WORKER -# ============================================ -OUTBOX_POLL_INTERVAL_MS=5000 # Poll every 5 seconds -OUTBOX_CLEANUP_INTERVAL_MS=60000 # Cleanup every 60 seconds -OUTBOX_BATCH_SIZE=100 # Process 100 entries per poll -OUTBOX_RETENTION_MS=300000 # Keep published entries for 5 minutes -OUTBOX_CLAIM_TIMEOUT_MS=30000 # Claim timeout for crashed workers - -# ============================================ -# RETRY & IDEMPOTENCY -# ============================================ -IDEMPOTENCY_TTL_SECONDS=86400 # Keep idempotency keys for 24 hours -MAX_RETRY_COUNT=5 # Max retries before marking as failed -PROCESSING_TTL_SECONDS=120 # Processing lock held for 2 minutes - -# ============================================ -# DELAYED NOTIFICATIONS -# ============================================ -DELAYED_POLL_INTERVAL_MS=1000 # Poll Redis every 1 second -DELAYED_BATCH_SIZE=10 # Fetch 10 due events per poll -MAX_POLLER_RETRIES=3 # Max retries before DLQ - -# ============================================ -# RECOVERY SERVICE -# ============================================ -RECOVERY_POLL_INTERVAL_MS=60000 # Run recovery every 1 minute -PROCESSING_STUCK_THRESHOLD_MS=300000 # Mark as stuck after 5 minutes -PENDING_STUCK_THRESHOLD_MS=300000 # Mark as orphaned after 5 minutes -RECOVERY_BATCH_SIZE=50 # Process 50 stuck notifications per poll -RECOVERY_CLAIM_TIMEOUT_MS=60000 # Claim timeout for recovery cron worker - -# ============================================ -# CLEANUP (runs as part of recovery) -# ============================================ -CLEANUP_RESOLVED_ALERTS_RETENTION_MS=86400000 # Keep resolved alerts for 24 hours -CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS=86400000 # Keep processed status outbox for 24 hours +REDIS_URL=redis://localhost:6379 # Redis connection URL (used for pub/sub sync & locks) # ============================================ # LOGGING (Grafana/Loki) # ============================================ -LOKI_URL= # Loki URL for log aggregation (http://host:port if self-hosted) -LOG_LEVEL=info # Log level: debug, info, warn, error -LOG_TO_FILE=true # Enable file logging +LOKI_URL=http://localhost:3100 # Optional Loki endpoint for centralized logging +LOG_LEVEL=info # Initial bootstrap log level (debug, info, warn, error) +LOG_TO_FILE=true # Enable file transport logging to /logs directory # ============================================ # ADMIN DASHBOARD # ============================================ -AUTH_SECRET= # Generate via: openssl rand -base64 32 -ADMIN_USERNAME=admin # Dashboard admin username -ADMIN_PASSWORD= # Dashboard admin password -AUTH_TRUST_HOST=true # Trust host for auth -API_BASE_URL= # Notification service API URL -WEBHOOK_HOST= # Webhook host for callbacks +AUTH_TRUST_HOST=true # Trust host header for authentication cookies +AUTH_SECRET= # Dashboard session signing secret (defaults to NS_API_KEY if omitted) +API_BASE_URL=http://localhost:3000 # Core Notification service API URL +WEBHOOK_HOST=dashboard # Webhook callback host WEBHOOK_PORT=3002 # Dashboard webhook port -DASHBOARD_PORT=3002 # Dashboard port -BASE_PATH= # Configurable BASE_PATH to render the admin dashboard -HTTPS_COOKIE= # Controls the storage of session cookies in http and https mode. (If set false, then the cookies can be stored in http connection) - - -# ============================================ -# PLUGIN SPECIFIC ENV VARIABLES (Partition variables + Plugin credentials) -# ============================================ - -# Channel-Specific Partitions (Dynamic - based on simplens.config.yaml channels) -# Topic: {CHANNEL}_notification, Env: {CHANNEL}_PARTITION -MOCK_PARTITION=1 # Partitions for mock_notification topic - -# PLUGIN CREDENTIALS (@simplens/nodemailer-gmail) (You need to install this plugin separately. Not included with simplens-core) -EMAIL_USER= # Gmail address -EMAIL_PASS= # Gmail app password (not regular password!) \ No newline at end of file +DASHBOARD_PORT=3002 # Next.js Dashboard port +BASE_PATH=/dashboard # Optional subpath (e.g. /dashboard) or empty for root +HTTPS_COOKIE=false # Controls cookie Secure flag ('true' for HTTPS prod, 'false' for local HTTP) \ No newline at end of file diff --git a/dashboard/.env.example b/dashboard/.env.example index 591ccd6..0ec52af 100644 --- a/dashboard/.env.example +++ b/dashboard/.env.example @@ -1,8 +1,6 @@ MONGO_URI= # mongodb connection uri PORT=3002 # port number -ADMIN_USERNAME= # admin username -ADMIN_PASSWORD= # admin password -AUTH_SECRET= # authentication secret +AUTH_SECRET= # authentication secret (optional: defaults to NS_API_KEY) NS_API_KEY= # notification service api key API_BASE_URL= # notification service base url WEBHOOK_HOST= # webhook host diff --git a/dashboard/app/api/auth/login/route.ts b/dashboard/app/api/auth/login/route.ts index f6dd3a5..118e11f 100644 --- a/dashboard/app/api/auth/login/route.ts +++ b/dashboard/app/api/auth/login/route.ts @@ -1,6 +1,8 @@ import { NextResponse } from "next/server"; +import axios from "axios"; import { createSession, validateCredentials } from "@/lib/session"; import { getBasePath } from "@/lib/utils"; +import { API_BASE_URL, NS_API_KEY } from "@/lib/api-config"; export async function POST(request: Request) { try { @@ -14,16 +16,48 @@ export async function POST(request: Request) { ); } - const result = validateCredentials(username, password); + let verifiedUser: { id: string; username: string } | null = null; - if (!result.isValid || !result.userId) { + // 1. Try verifying via backend Core API (MongoDB credentials) + try { + const response = await axios.post( + `${API_BASE_URL}/api/admin/auth/verify`, + { username, password }, + { + headers: { + Authorization: `Bearer ${NS_API_KEY}`, + "Content-Type": "application/json", + }, + timeout: 5000, + } + ); + + if (response.data?.isValid && response.data?.user) { + verifiedUser = response.data.user; + } + } catch (apiErr: unknown) { + // If backend returned 401, it's explicitly invalid credentials + if (axios.isAxiosError(apiErr) && apiErr.response?.status === 401) { + return NextResponse.json( + { error: "Invalid username or password" }, + { status: 401 } + ); + } + // Fallback for network issues / local dev without backend running + const fallbackResult = validateCredentials(username, password); + if (fallbackResult.isValid && fallbackResult.userId) { + verifiedUser = { id: fallbackResult.userId, username }; + } + } + + if (!verifiedUser) { return NextResponse.json( { error: "Invalid username or password" }, { status: 401 } ); } - await createSession(result.userId, username); + await createSession(verifiedUser.id, verifiedUser.username); const basePath = getBasePath(); return NextResponse.json({ diff --git a/dashboard/app/api/auth/setup/route.ts b/dashboard/app/api/auth/setup/route.ts new file mode 100644 index 0000000..7ea7002 --- /dev/null +++ b/dashboard/app/api/auth/setup/route.ts @@ -0,0 +1,70 @@ +import { NextResponse } from "next/server"; +import axios from "axios"; +import { API_BASE_URL, NS_API_KEY } from "@/lib/api-config"; +import { createSession } from "@/lib/session"; +import { getBasePath } from "@/lib/utils"; + +export async function POST(request: Request) { + try { + const body = await request.json(); + const { username, password } = body; + + if (!username || !password) { + return NextResponse.json( + { error: "Username and password are required" }, + { status: 400 } + ); + } + + const trimmedUsername = String(username).trim(); + if (trimmedUsername.length < 3 || trimmedUsername.length > 50) { + return NextResponse.json( + { error: "Username must be between 3 and 50 characters" }, + { status: 400 } + ); + } + + if (password.length < 8 || password.length > 64) { + return NextResponse.json( + { error: "Password must be between 8 and 64 characters" }, + { status: 400 } + ); + } + + const response = await axios.post( + `${API_BASE_URL}/api/admin/auth/setup`, + { username, password }, + { + headers: { + Authorization: `Bearer ${NS_API_KEY}`, + "Content-Type": "application/json", + }, + timeout: 5000, + } + ); + + if (response.data.success && response.data.user) { + await createSession(response.data.user.id, response.data.user.username); + } + + const basePath = getBasePath(); + return NextResponse.json({ + success: true, + redirectUrl: `${basePath}/dashboard`, + }); + } catch (error: unknown) { + console.error("Admin setup error:", error); + if (axios.isAxiosError(error) && error.response?.data) { + const data = error.response.data as { error?: string; details?: unknown }; + return NextResponse.json( + { error: data.error || "Setup failed", details: data.details }, + { status: error.response.status || 400 } + ); + } + + return NextResponse.json( + { error: "An unexpected error occurred during setup" }, + { status: 500 } + ); + } +} diff --git a/dashboard/app/api/auth/status/route.ts b/dashboard/app/api/auth/status/route.ts new file mode 100644 index 0000000..bad55d4 --- /dev/null +++ b/dashboard/app/api/auth/status/route.ts @@ -0,0 +1,21 @@ +import { NextResponse } from "next/server"; +import axios from "axios"; +import { API_BASE_URL, NS_API_KEY } from "@/lib/api-config"; + +export async function GET() { + try { + const response = await axios.get(`${API_BASE_URL}/api/admin/auth/status`, { + headers: { + Authorization: `Bearer ${NS_API_KEY}`, + }, + timeout: 5000, + }); + + return NextResponse.json(response.data); + } catch (error) { + console.error("Error checking auth status from backend:", error); + // Fallback check against process.env + const hasEnv = Boolean(process.env.ADMIN_USERNAME && process.env.ADMIN_PASSWORD); + return NextResponse.json({ isConfigured: hasEnv }); + } +} diff --git a/dashboard/app/login/page.tsx b/dashboard/app/login/page.tsx index e640518..a831855 100644 --- a/dashboard/app/login/page.tsx +++ b/dashboard/app/login/page.tsx @@ -11,7 +11,7 @@ import { AlertTriangle, Eye, EyeOff, Loader2, Lock, User } from "lucide-react"; import { ElegantShape } from "@/components/elegant-shape"; import { withBasePath } from "@/lib/utils"; import { authService } from "@/lib/api-client"; -import { useState } from "react"; +import { useEffect, useState } from "react"; export default function LoginPage() { const router = useRouter(); @@ -21,6 +21,21 @@ export default function LoginPage() { const [isLoading, setIsLoading] = useState(false); const [showPassword, setShowPassword] = useState(false); + useEffect(() => { + const checkStatus = async () => { + try { + const status = await authService.getAuthStatus(); + if (!status.isConfigured) { + router.replace(withBasePath("/setup")); + } + } catch (err) { + console.warn("Failed to check auth configuration status on login page", err); + } + }; + + checkStatus(); + }, [router]); + const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); setError(""); @@ -30,8 +45,7 @@ export default function LoginPage() { const data = await authService.login({ username, password }); setIsLoading(false); // Redirect to dashboard - router.push(data.redirectUrl || withBasePath(`/dashboard`)); - router.refresh(); + window.location.href = data.redirectUrl || withBasePath(`/dashboard`); return; } catch (err) { const error = err as Error; diff --git a/dashboard/app/settings/page.tsx b/dashboard/app/settings/page.tsx index 293d94f..48e4eb5 100644 --- a/dashboard/app/settings/page.tsx +++ b/dashboard/app/settings/page.tsx @@ -1,30 +1,133 @@ "use client"; +import { useEffect, useState } from "react"; import { DashboardLayout } from "@/components/layout/dashboard-layout"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Separator } from "@/components/ui/separator"; import { Badge } from "@/components/ui/badge"; import { Switch } from "@/components/ui/switch"; import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { useGlassmorphism } from "@/components/glassmorphism-provider"; import { useTour } from "@/components/tour/tour-provider"; -import { RotateCcw } from "lucide-react"; +import { + ChevronDown, + Clock, + Cpu, + Database, + Layers, + Loader2, + RefreshCw, + RotateCcw, + Save, + ShieldAlert, + Sliders, + Sparkles, +} from "lucide-react"; +import { settingsService } from "@/lib/api-client"; +import type { OperationalSettings } from "@/lib/types"; +import { toast } from "sonner"; import packageJson from "../../package.json"; export default function SettingsPage() { const { enabled, setEnabled } = useGlassmorphism(); const { restartTour } = useTour(); + const [settings, setSettings] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [isSaving, setIsSaving] = useState(false); + const [isResetting, setIsResetting] = useState(false); + const [showAdvanced, setShowAdvanced] = useState(false); + + // Fetch operational settings on mount + useEffect(() => { + const fetchSettings = async () => { + try { + const res = await settingsService.get(); + if (res.success && res.settings) { + setSettings(res.settings); + } + } catch (err) { + console.error("Failed to load settings", err); + toast.error("Failed to load system settings from server"); + } finally { + setIsLoading(false); + } + }; + + fetchSettings(); + }, []); + + const handleSave = async () => { + if (!settings) return; + setIsSaving(true); + try { + const res = await settingsService.update(settings); + if (res.success && res.settings) { + setSettings(res.settings); + toast.success("Settings updated and synchronized across all instances!"); + } + } catch (err: unknown) { + const errorObj = err as Error; + toast.error(errorObj.message || "Failed to update settings"); + } finally { + setIsSaving(false); + } + }; + + const handleReset = async () => { + if (!confirm("Are you sure you want to reset all operational settings to system defaults?")) { + return; + } + + setIsResetting(true); + try { + const res = await settingsService.reset(); + if (res.success && res.settings) { + setSettings(res.settings); + toast.success("Operational settings reset to system defaults!"); + } + } catch (err: unknown) { + const errorObj = err as Error; + toast.error(errorObj.message || "Failed to reset settings"); + } finally { + setIsResetting(false); + } + }; + + const updateGroupField = ( + group: keyof OperationalSettings, + field: string, + value: unknown + ) => { + if (!settings) return; + setSettings({ + ...settings, + [group]: { + ...settings[group], + [field]: value, + }, + }); + }; + return ( -
- - - Appearance - Visual effects and UI rendering +
+ {/* 1. Appearance & Tour */} + + + + + Appearance & Tour + + + Visual styling layers and interactive guided walkthroughs +
@@ -54,7 +157,7 @@ export default function SettingsPage() { variant="outline" size="sm" onClick={restartTour} - className="gap-2" + className="gap-2 text-xs" > Restart Tour @@ -63,48 +166,574 @@ export default function SettingsPage() { - - {/* System Info */} - - - System Information - Current system configuration + {/* 2. System Info */} + + + + + System Information + + Current host deployment details
- Dashboard Version + Dashboard Version {packageJson.version}
- Environment + Runtime Mode {process.env.NODE_ENV === "production" ? "Production" : "Development"}
- Database - MongoDB + Configuration Store + + MongoDB (system_configs) & Redis Pub/Sub +
- {/* About */} - - - About SimpleNS - Notification service information - - -

- SimpleNS is an open-source, self-hosted notification orchestration engine that manages - delivery workflows—retries, scheduling, crash recovery, and scaling—while delegating - the actual sending to plugins. Build your own providers or use community plugins to - support any channel: Email, SMS, WhatsApp, Push, and beyond. -

- - + {/* 3. Advanced Settings (Operational Engine Configurations) */} +
+ + + {showAdvanced && ( +
+ {/* Action Bar inside Advanced Settings */} +
+
+

Live Engine Tuning

+

+ Changes synchronize immediately across all running containers without service restarts. +

+
+
+ + +
+
+ + {isLoading ? ( +
+ +

Loading system settings...

+
+ ) : !settings ? ( +
+

Failed to load configuration.

+ +
+ ) : ( +
+ {/* 1. Outbox Background Worker */} + + + + + Outbox Worker + + + Configures the background outbox polling, batch processing, and event cleanup. + + + +
+
+ + + updateGroupField( + "worker", + "outbox_poll_interval_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Safe: 500 - 60,000 +
+ +
+ + + updateGroupField( + "worker", + "outbox_batch_size", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Safe: 1 - 2,000 +
+
+ +
+
+ + + updateGroupField( + "worker", + "outbox_cleanup_interval_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Safe: 5k - 600k +
+ +
+ + + updateGroupField( + "worker", + "outbox_retention_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Published retention +
+
+ +
+ + + updateGroupField( + "worker", + "outbox_claim_timeout_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Threshold before an abandoned lock is reclaimed by peers + +
+ + + + {/* 2. Retry & Idempotency */} + + + + + Retry & Idempotency + + + Failure retries, deduplication caching, and execution locks. + + + +
+ + + updateGroupField( + "retry", + "max_retry_count", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Max attempts before permanently marking as failed + +
+ +
+ + + updateGroupField( + "retry", + "idempotency_ttl_seconds", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + How long deduplication keys are held in Redis (86400s = 24h) + +
+ +
+ + + updateGroupField( + "retry", + "processing_ttl_seconds", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Active processing mutex duration (default: 120s) + +
+ + + + {/* 3. Delayed Processor */} + + + + + Delayed Processor + + + Parameters for scheduled and deferred notification poller. + + + +
+
+ + + updateGroupField( + "delayed", + "delayed_poll_interval_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Safe: 200 - 30,000 +
+ +
+ + + updateGroupField( + "delayed", + "delayed_batch_size", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Safe: 1 - 500 +
+
+ +
+ + + updateGroupField( + "delayed", + "max_poller_retries", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Attempts before moving an unroutable scheduled event to DLQ + +
+ + + + {/* 4. Recovery & Health Service */} + + + + + Recovery & Audits + + + Detects orphaned notifications, resolves stuck states, and manages cleanups. + + + +
+
+ + + updateGroupField( + "recovery", + "recovery_poll_interval_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 60,000 +
+ +
+ + + updateGroupField( + "recovery", + "recovery_batch_size", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 50 +
+
+ +
+
+ + + updateGroupField( + "recovery", + "processing_stuck_threshold_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 300,000 +
+ +
+ + + updateGroupField( + "recovery", + "pending_stuck_threshold_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 300,000 +
+
+ + + + {/* 5. System, API & Logging */} + + + + + API & Observability + + + API request boundaries and Winston logger dynamics across all containers. + + + +
+
+ + + updateGroupField( + "api", + "max_batch_req_limit", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Max items per batch notification API call + +
+ +
+ + + + Immediately switches log level with zero restarts + +
+ +
+
+ +

+ Store rotating JSON log files in /logs directory +

+
+
+ + updateGroupField("logging", "log_to_file", val) + } + /> +
+
+
+ + +
+ )} +
+ )} +
); diff --git a/dashboard/app/setup/page.tsx b/dashboard/app/setup/page.tsx new file mode 100644 index 0000000..67f4e6e --- /dev/null +++ b/dashboard/app/setup/page.tsx @@ -0,0 +1,381 @@ +"use client"; + +import { useRouter } from "next/navigation"; +import { motion } from "motion/react"; +import Image from "next/image"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { AlertTriangle, ArrowRight, Check, Eye, EyeOff, KeyRound, Loader2, Lock, Radio, ShieldCheck, User, Zap } from "lucide-react"; +import { ElegantShape } from "@/components/elegant-shape"; +import { withBasePath } from "@/lib/utils"; +import { authService } from "@/lib/api-client"; +import { useEffect, useState } from "react"; + +export default function SetupPage() { + const router = useRouter(); + const [username, setUsername] = useState("admin"); + const [password, setPassword] = useState(""); + const [confirmPassword, setConfirmPassword] = useState(""); + const [error, setError] = useState(""); + const [isLoading, setIsLoading] = useState(false); + const [isCheckingStatus, setIsCheckingStatus] = useState(true); + const [showPassword, setShowPassword] = useState(false); + const [showConfirmPassword, setShowConfirmPassword] = useState(false); + + // On mount, check if admin credentials have already been configured + useEffect(() => { + const checkStatus = async () => { + try { + const status = await authService.getAuthStatus(); + if (status.isConfigured) { + router.replace(withBasePath("/login")); + return; + } + } catch (err) { + console.warn("Failed to check auth configuration status", err); + } finally { + setIsCheckingStatus(false); + } + }; + + checkStatus(); + }, [router]); + + // Password strength calculation + const hasMinLength = password.length >= 8; + const hasMaxLength = password.length <= 64; + const hasUpper = /[A-Z]/.test(password); + const hasLower = /[a-z]/.test(password); + const hasNumber = /[0-9]/.test(password); + const hasSpecial = /[^A-Za-z0-9]/.test(password); + + const score = [ + hasMinLength && hasMaxLength, + hasUpper && hasLower, + hasNumber, + hasSpecial, + ].filter(Boolean).length; + + const strengthLabel = + password.length === 0 ? "" : + score <= 1 ? "Weak" : + score === 2 ? "Fair" : + score === 3 ? "Good" : "Strong"; + + const strengthBarColor = + score <= 1 ? "bg-red-500" : + score === 2 ? "bg-amber-500" : + score === 3 ? "bg-blue-500" : "bg-emerald-500"; + + const strengthTextColor = + score <= 1 ? "text-red-500 dark:text-red-400" : + score === 2 ? "text-amber-500 dark:text-amber-400" : + score === 3 ? "text-blue-500 dark:text-blue-400" : "text-emerald-500 dark:text-emerald-400"; + + const passwordsMatch = password.length > 0 && password === confirmPassword; + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + setError(""); + + const trimmedUser = username.trim(); + if (trimmedUser.length < 3 || trimmedUser.length > 50) { + setError("Username must be between 3 and 50 characters."); + return; + } + + if (password.length < 8) { + setError("Password must be at least 8 characters long."); + return; + } + + if (password.length > 64) { + setError("Password cannot exceed 64 characters."); + return; + } + + if (password !== confirmPassword) { + setError("Passwords do not match."); + return; + } + + setIsLoading(true); + + try { + const data = await authService.setup({ username: trimmedUser, password }); + setIsLoading(false); + window.location.href = data.redirectUrl || withBasePath("/dashboard"); + return; + } catch (err) { + const errorObj = err as Error; + setError(errorObj.message || "An error occurred during setup. Please try again."); + setIsLoading(false); + } + }; + + if (isCheckingStatus) { + return ( +
+ +
+ ); + } + + return ( +
+ {/* Left Column: Brand Showcase Panel */} +
+ {/* Background Ambient Shapes */} +
+ + +
+ + {/* Top Logo & Brand Tag */} +
+
+ SimpleNS Logo +
+
+ + First-Time Workspace Onboarding +
+
+ + {/* Center Hero Copy */} +
+

+ Scalable, multi-channel notification engine. +

+

+ Orchestrate email, SMS, push, and webhooks with guaranteed delivery, delayed queue scheduling, and live zero-downtime reconfiguration. +

+ +
+
+
+ +
+ Instant zero-downtime hot-reload across all services +
+
+
+ +
+ Unified multi-channel routing and dead-letter recovery +
+
+
+ + {/* Bottom Footer Note */} +
+ SimpleNS • Fast, Resilient, Open Architecture +
+
+ + {/* Right Column: Clean Elegant Setup Form */} +
+ + {/* Mobile Logo */} +
+ SimpleNS Logo +
+ Initial Setup +
+
+ + {/* Header */} +
+

+ Create Administrator +

+

+ Set up your master credentials to secure and manage this SimpleNS instance. +

+
+ + {/* Form */} +
+ {error && ( +
+ + {error} +
+ )} + + {/* Username */} +
+ + setUsername(e.target.value)} + required + minLength={3} + maxLength={50} + disabled={isLoading} + className="h-11 bg-background text-sm font-medium" + /> +

+ Between 3 and 50 characters. +

+
+ + {/* Password */} +
+ +
+ setPassword(e.target.value)} + required + minLength={8} + maxLength={64} + disabled={isLoading} + className="h-11 bg-background text-sm font-medium pr-10" + /> + +
+ + {/* Minimalist Password Strength Bar */} + {password.length > 0 && ( +
+
+ Password strength + + {strengthLabel} + +
+ {/* Continuous sleek bar */} +
+
+
+
+ )} + +

+ 8 to 64 characters with a mix of letters and numbers. +

+
+ + {/* Confirm Password */} +
+ +
+ setConfirmPassword(e.target.value)} + required + minLength={8} + maxLength={64} + disabled={isLoading} + className="h-11 bg-background text-sm font-medium pr-10" + /> + +
+ {confirmPassword.length > 0 && ( +

+ {passwordsMatch ? ( + <> + Passwords match + + ) : ( + "Passwords do not match" + )} +

+ )} +
+ + {/* Submit Button */} + + + +
+
+ ); +} diff --git a/dashboard/lib/api-client.ts b/dashboard/lib/api-client.ts index 68e93de..d085872 100644 --- a/dashboard/lib/api-client.ts +++ b/dashboard/lib/api-client.ts @@ -23,6 +23,8 @@ import { PluginCatalogItem, ProviderDto, ChannelRoutingDto, + OperationalSettings, + AdminAuthStatus, } from './types'; export class ApiError extends Error { @@ -78,6 +80,19 @@ export const authService = { logout: (): Promise => apiClient.post('/api/auth/logout'), getSession: (): Promise<{ authenticated: boolean; user?: { id: string; username: string } }> => apiClient.get('/api/auth/session'), + getAuthStatus: (): Promise => + apiClient.get('/api/auth/status'), + setup: (payload: { username: string; password: string }): Promise<{ success: boolean; redirectUrl: string; message?: string }> => + apiClient.post('/api/auth/setup', payload), +}; + +export const settingsService = { + get: (): Promise<{ success: boolean; settings: OperationalSettings }> => + apiClient.get('/api/settings'), + update: (payload: Partial): Promise<{ success: boolean; message: string; settings: OperationalSettings }> => + apiClient.put('/api/settings', payload), + reset: (): Promise<{ success: boolean; message: string; settings: OperationalSettings }> => + apiClient.post('/api/settings/reset'), }; export const notificationService = { diff --git a/dashboard/lib/session.ts b/dashboard/lib/session.ts index 6f6fd3e..d213e9c 100644 --- a/dashboard/lib/session.ts +++ b/dashboard/lib/session.ts @@ -12,14 +12,14 @@ interface SessionPayload { } /** - * Get the auth secret from environment, throws if not configured + * Get the auth secret from environment, with safe deterministic fallbacks */ function getAuthSecret(): string { - const secret = process.env.AUTH_SECRET; - if (!secret) { - throw new Error("AUTH_SECRET environment variable is not configured"); + const secret = process.env.AUTH_SECRET || process.env.JWT_SECRET || process.env.NS_API_KEY; + if (secret && secret.trim().length > 0) { + return secret.trim(); } - return secret; + return "simplens-session-auth-secret-fallback-key-32b"; } /** @@ -89,7 +89,7 @@ export async function createSession(userId: string, username: string): Promise 0) { + return secret.trim(); } - return secret; + return "simplens-session-auth-secret-fallback-key-32b"; } // Web Crypto API compatible HMAC-SHA256 diff --git a/src/api/controllers/admin-auth.controller.ts b/src/api/controllers/admin-auth.controller.ts new file mode 100644 index 0000000..df8bb17 --- /dev/null +++ b/src/api/controllers/admin-auth.controller.ts @@ -0,0 +1,152 @@ +/** + * Admin Authentication & Initial Setup Controller + * + * Manages first-time administrator account onboarding, authentication status checks, + * and credential verification using crypto.scrypt hashing and constant-time comparison. + */ + +import { Request, Response } from 'express'; +import { randomBytes, scryptSync, timingSafeEqual } from 'crypto'; +import system_config_model from '@src/database/models/system-config.models.js'; +import { adminSetupSchema, adminLoginSchema } from '@src/types/schemas.js'; +import type { AdminCredentialsDoc } from '@src/types/types.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +export const ADMIN_CREDENTIALS_KEY = 'admin_credentials'; + +/** + * Check if the administrator credentials have been configured + * GET /api/admin/auth/status + */ +export const getAuthStatus = async (_req: Request, res: Response): Promise => { + try { + const doc = await system_config_model.findOne({ key: ADMIN_CREDENTIALS_KEY }); + const isConfigured = Boolean(doc?.value); + + res.status(200).json({ isConfigured }); + } catch (err) { + logger.error('Failed to query admin auth status', err); + res.status(500).json({ error: 'Internal server error checking admin auth status' }); + } +}; + +/** + * Configure administrator account on initial setup + * POST /api/admin/auth/setup + */ +export const setupAdmin = async (req: Request, res: Response): Promise => { + try { + const validation = adminSetupSchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + error: 'Validation failed', + details: validation.error.flatten().fieldErrors, + }); + return; + } + + const { username, password } = validation.data; + + // Check if already configured + const existingDoc = await system_config_model.findOne({ key: ADMIN_CREDENTIALS_KEY }); + if (existingDoc?.value) { + res.status(400).json({ + error: 'Administrator credentials have already been configured. Initial setup is locked.', + }); + return; + } + + // Generate cryptographic salt and scrypt hash + const salt = randomBytes(32).toString('hex'); + const password_hash = scryptSync(password, salt, 64).toString('hex'); + + const credentialsData: AdminCredentialsDoc = { + username, + password_hash, + salt, + created_at: new Date().toISOString(), + }; + + await system_config_model.create({ + key: ADMIN_CREDENTIALS_KEY, + value: credentialsData, + }); + + logger.success(`Administrator account configured successfully for username: ${username}`); + + res.status(201).json({ + success: true, + message: 'Administrator account configured successfully.', + user: { + id: 'admin-1', + username, + }, + }); + } catch (err) { + logger.error('Error during administrator setup', err); + res.status(500).json({ error: 'Failed to configure administrator account' }); + } +}; + +/** + * Verify administrator credentials + * POST /api/admin/auth/verify + */ +export const verifyAdmin = async (req: Request, res: Response): Promise => { + try { + const validation = adminLoginSchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + isValid: false, + message: 'Username and password are required', + }); + return; + } + + const { username, password } = validation.data; + + const doc = await system_config_model.findOne({ key: ADMIN_CREDENTIALS_KEY }); + + if (doc?.value) { + const credentials = doc.value as AdminCredentialsDoc; + + // Username match check + if (credentials.username !== username) { + res.status(401).json({ isValid: false, message: 'Invalid username or password' }); + return; + } + + // Compute scrypt hash with stored salt + const computedHash = scryptSync(password, credentials.salt, 64).toString('hex'); + const storedHashBuf = Buffer.from(credentials.password_hash, 'hex'); + const computedHashBuf = Buffer.from(computedHash, 'hex'); + + if (storedHashBuf.length === computedHashBuf.length && timingSafeEqual(storedHashBuf, computedHashBuf)) { + res.status(200).json({ + isValid: true, + user: { id: 'admin-1', username: credentials.username }, + }); + return; + } + + res.status(401).json({ isValid: false, message: 'Invalid username or password' }); + return; + } + + // Fallback check against process.env if present + const envUser = process.env.ADMIN_USERNAME; + const envPass = process.env.ADMIN_PASSWORD; + if (envUser && envPass && username === envUser && password === envPass) { + res.status(200).json({ + isValid: true, + user: { id: 'admin-1', username: envUser }, + }); + return; + } + + res.status(401).json({ isValid: false, message: 'Invalid username or password' }); + } catch (err) { + logger.error('Error verifying administrator credentials', err); + res.status(500).json({ error: 'Failed to verify administrator credentials' }); + } +}; diff --git a/src/api/controllers/settings.controller.ts b/src/api/controllers/settings.controller.ts new file mode 100644 index 0000000..80d7786 --- /dev/null +++ b/src/api/controllers/settings.controller.ts @@ -0,0 +1,76 @@ +/** + * Dynamic Operational Settings Controller + * + * Handles reading, updating, and resetting operational configuration settings. + */ + +import { Request, Response } from 'express'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; +import { partialOperationalSettingsSchema } from '@src/types/schemas.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +/** + * Get current operational settings + * GET /api/settings + */ +export const getSettings = async (_req: Request, res: Response): Promise => { + try { + const settings = dynamicConfig.getAll(); + res.status(200).json({ + success: true, + settings, + }); + } catch (err) { + logger.error('Failed to retrieve operational settings', err); + res.status(500).json({ error: 'Failed to retrieve operational settings' }); + } +}; + +/** + * Update operational settings + * PUT /api/settings + */ +export const updateSettings = async (req: Request, res: Response): Promise => { + try { + const validation = partialOperationalSettingsSchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + error: 'Validation failed', + details: validation.error.flatten().fieldErrors, + }); + return; + } + + const updatedSettings = await dynamicConfig.updateSettings(validation.data); + logger.success('Operational settings updated and broadcasted'); + + res.status(200).json({ + success: true, + message: 'Operational settings updated and broadcasted successfully', + settings: updatedSettings, + }); + } catch (err) { + logger.error('Failed to update operational settings', err); + res.status(500).json({ error: 'Failed to update operational settings' }); + } +}; + +/** + * Reset operational settings to system defaults + * POST /api/settings/reset + */ +export const resetSettings = async (_req: Request, res: Response): Promise => { + try { + const reset = await dynamicConfig.resetSettings(); + logger.info('Operational settings reset to system defaults and broadcasted'); + + res.status(200).json({ + success: true, + message: 'Operational settings reset to system defaults successfully', + settings: reset, + }); + } catch (err) { + logger.error('Failed to reset operational settings', err); + res.status(500).json({ error: 'Failed to reset operational settings' }); + } +}; diff --git a/src/api/routes/admin-auth.routes.ts b/src/api/routes/admin-auth.routes.ts new file mode 100644 index 0000000..b79d082 --- /dev/null +++ b/src/api/routes/admin-auth.routes.ts @@ -0,0 +1,19 @@ +/** + * Admin Authentication & Setup Routes + */ + +import { Router } from 'express'; +import { getAuthStatus, setupAdmin, verifyAdmin } from '../controllers/admin-auth.controller.js'; + +const router = Router(); + +// GET /api/admin/auth/status - Check if admin credentials are configured +router.get('/status', getAuthStatus); + +// POST /api/admin/auth/setup - Initial administrator configuration +router.post('/setup', setupAdmin); + +// POST /api/admin/auth/verify - Verify administrator credentials +router.post('/verify', verifyAdmin); + +export default router; diff --git a/src/api/routes/settings.routes.ts b/src/api/routes/settings.routes.ts new file mode 100644 index 0000000..4d6a75d --- /dev/null +++ b/src/api/routes/settings.routes.ts @@ -0,0 +1,19 @@ +/** + * Dynamic Operational Settings Routes + */ + +import { Router } from 'express'; +import { getSettings, updateSettings, resetSettings } from '../controllers/settings.controller.js'; + +const router = Router(); + +// GET /api/settings - Retrieve current active settings +router.get('/', getSettings); + +// PUT /api/settings - Update operational settings +router.put('/', updateSettings); + +// POST /api/settings/reset - Reset operational settings to system defaults +router.post('/reset', resetSettings); + +export default router; diff --git a/src/api/server.ts b/src/api/server.ts index c03e256..6ea5244 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -12,6 +12,9 @@ import notification_templates_router from '@src/api/routes/notification_template import notifications_management_router from './routes/notifications-management.routes.js'; import alerts_router from './routes/alerts.routes.js'; import dashboard_router from './routes/dashboard.routes.js'; +import settings_router from './routes/settings.routes.js'; +import admin_auth_router from './routes/admin-auth.routes.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { auth_middleware } from './middlewares/auth_middleware.js'; import http from 'http'; import helmet from 'helmet'; @@ -67,12 +70,17 @@ app.use('/api/providers', auth_middleware, providers_router); app.use('/api/channels/routing', auth_middleware, channel_routing_router); app.use('/api/admin-channels', auth_middleware, admin_channels_router); app.use('/api/templates', auth_middleware, notification_templates_router); +app.use('/api/settings', auth_middleware, settings_router); +app.use('/api/admin/auth', admin_auth_router); const start_server = async () => { try { const db = await connectMongoDB(); logger.success("Successfully connected to MongoDB"); + // 0. Initialize Dynamic Configuration & seed defaults into MongoDB + await dynamicConfig.initialize(true); + // 1. Initialize Redis subscriber & sync handlers for plugins await PluginSyncService.startSubscriber(); PluginManagerService.registerSyncHandlers(); diff --git a/src/config/dynamic-config.service.ts b/src/config/dynamic-config.service.ts new file mode 100644 index 0000000..592e18c --- /dev/null +++ b/src/config/dynamic-config.service.ts @@ -0,0 +1,264 @@ +/** + * Dynamic Operational Configuration Service + * + * Manages runtime system settings stored in MongoDB and live hot-reloaded + * across all running containers via Redis Pub/Sub without process restarts. + */ + +import { EventEmitter } from 'events'; +import system_config_model from '@src/database/models/system-config.models.js'; +import { getRedisClient } from './redis.config.js'; +import { configSyncLogger as logger, setLogLevel } from '@src/workers/utils/logger.js'; +import { operationalSettingsSchema } from '@src/types/schemas.js'; +import type { + OperationalSettings, + PartialOperationalSettings, + SystemConfigSyncAction, + SystemConfigSyncMessage, +} from '@src/types/types.js'; +import type { Redis as RedisType } from 'ioredis'; + +export const CONFIG_SYNC_CHANNEL = 'simplens:system:config:sync'; + +export const SYSTEM_DEFAULT_OPERATIONAL_SETTINGS: OperationalSettings = { + api: { + max_batch_req_limit: 1000, + }, + worker: { + outbox_poll_interval_ms: 5000, + outbox_cleanup_interval_ms: 60000, + outbox_batch_size: 100, + outbox_retention_ms: 300000, + outbox_claim_timeout_ms: 30000, + }, + retry: { + max_retry_count: 5, + idempotency_ttl_seconds: 86400, + processing_ttl_seconds: 120, + }, + delayed: { + delayed_poll_interval_ms: 1000, + delayed_batch_size: 10, + max_poller_retries: 3, + }, + recovery: { + recovery_poll_interval_ms: 60000, + processing_stuck_threshold_ms: 300000, + pending_stuck_threshold_ms: 300000, + recovery_batch_size: 50, + recovery_claim_timeout_ms: 60000, + cleanup_resolved_alerts_retention_ms: 86400000, + cleanup_processed_status_outbox_retention_ms: 86400000, + }, + logging: { + log_level: 'info', + log_to_file: true, + }, +}; + +export class DynamicConfigService extends EventEmitter { + private static instance: DynamicConfigService; + private settings: OperationalSettings = { ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS }; + private isInitialized = false; + private subClient: RedisType | null = null; + private instanceId: string; + + private constructor() { + super(); + this.instanceId = process.env.HOSTNAME || process.env.WORKER_ID || `instance-${Math.random().toString(36).slice(2, 9)}`; + } + + public static getInstance(): DynamicConfigService { + if (!DynamicConfigService.instance) { + DynamicConfigService.instance = new DynamicConfigService(); + } + return DynamicConfigService.instance; + } + + /** + * Initialize DynamicConfigService + * @param isApiServer - If true, seeds MongoDB with default values if not present, and broadcasts. + */ + public async initialize(isApiServer: boolean = false): Promise { + if (this.isInitialized) return; + + // 1. Subscribe to Redis sync updates + await this.subscribeToUpdates(); + + // 2. Fetch from MongoDB or seed defaults + try { + const doc = await system_config_model.findOne({ key: 'operational_settings' }); + + if (doc?.value) { + const parsed = operationalSettingsSchema.safeParse(doc.value); + if (parsed.success) { + this.settings = parsed.data; + logger.info('Loaded operational settings from MongoDB'); + } else { + logger.warn('Failed to parse operational settings from MongoDB, using defaults', { error: parsed.error }); + this.settings = { ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS }; + } + } else if (isApiServer) { + this.settings = { ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS }; + await system_config_model.create({ + key: 'operational_settings', + value: this.settings, + }); + logger.success('Seeded MongoDB with default operational settings'); + + await this.publishSync('SETTINGS_SEEDED'); + } else { + this.settings = { ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS }; + logger.info('No operational settings found in MongoDB yet; using default in-memory settings'); + } + } catch (err) { + logger.error('Error loading operational settings from MongoDB, using defaults', err); + this.settings = { ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS }; + } + + // Apply log level to all instantiated loggers + setLogLevel(this.settings.logging.log_level); + + this.isInitialized = true; + } + + public get(group: K): OperationalSettings[K] { + return this.settings[group]; + } + + public getAll(): OperationalSettings { + return { ...this.settings }; + } + + public async updateSettings(newSettings: PartialOperationalSettings): Promise { + const merged = { + api: { ...this.settings.api, ...(newSettings.api || {}) }, + worker: { ...this.settings.worker, ...(newSettings.worker || {}) }, + retry: { ...this.settings.retry, ...(newSettings.retry || {}) }, + delayed: { ...this.settings.delayed, ...(newSettings.delayed || {}) }, + recovery: { ...this.settings.recovery, ...(newSettings.recovery || {}) }, + logging: { ...this.settings.logging, ...(newSettings.logging || {}) }, + }; + + const validated = operationalSettingsSchema.parse(merged); + + await system_config_model.findOneAndUpdate( + { key: 'operational_settings' }, + { value: validated }, + { upsert: true, returnDocument: 'after' } + ); + + this.settings = validated; + setLogLevel(this.settings.logging.log_level); + + this.emit('change', this.settings); + + // Broadcast sync event to all instances via Redis Pub/Sub + await this.publishSync('SETTINGS_UPDATED', newSettings); + + return this.settings; + } + + public async resetSettings(): Promise { + const validated = operationalSettingsSchema.parse(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS); + + await system_config_model.findOneAndUpdate( + { key: 'operational_settings' }, + { value: validated }, + { upsert: true, returnDocument: 'after' } + ); + + this.settings = validated; + setLogLevel(this.settings.logging.log_level); + + this.emit('change', this.settings); + + await this.publishSync('SETTINGS_RESET'); + + return this.settings; + } + + public async loadFromDatabase(): Promise { + try { + const doc = await system_config_model.findOne({ key: 'operational_settings' }); + if (doc?.value) { + const parsed = operationalSettingsSchema.safeParse(doc.value); + if (parsed.success) { + this.settings = parsed.data; + setLogLevel(this.settings.logging.log_level); + this.emit('change', this.settings); + } + } + } catch (err) { + logger.error('Failed to reload operational settings from MongoDB', err); + } + } + + private async publishSync(action: SystemConfigSyncAction, payload?: PartialOperationalSettings): Promise { + try { + const redis = getRedisClient(); + const message: SystemConfigSyncMessage = { + action, + source_instance_id: this.instanceId, + timestamp: new Date().toISOString(), + payload, + }; + await redis.publish(CONFIG_SYNC_CHANNEL, JSON.stringify(message)); + logger.info(`Published config sync event [${action}] to ${CONFIG_SYNC_CHANNEL}`); + } catch (err) { + logger.error(`Failed to broadcast settings sync event [${action}] to Redis`, err); + } + } + + private async subscribeToUpdates(): Promise { + try { + const baseClient = getRedisClient(); + this.subClient = baseClient.duplicate(); + + // If client status is not ready/connecting, connect + if (this.subClient.status !== 'ready' && this.subClient.status !== 'connecting') { + await this.subClient.connect().catch(() => {}); + } + + await this.subClient.subscribe(CONFIG_SYNC_CHANNEL); + + this.subClient.on('message', async (channel, messageStr) => { + if (channel !== CONFIG_SYNC_CHANNEL) return; + + try { + const message: SystemConfigSyncMessage = JSON.parse(messageStr); + + // Skip self-originated updates + if (message.source_instance_id === this.instanceId) { + return; + } + + logger.info(`Received config sync event [${message.action}] from ${message.source_instance_id}`); + await this.loadFromDatabase(); + } catch (err) { + logger.error('Failed to process settings sync message from Redis', err); + } + }); + } catch (err) { + logger.warn('Failed to subscribe to Redis config sync channel, running with in-memory settings', { + error: err instanceof Error ? err.message : String(err) + }); + } + } + + public async close(): Promise { + if (this.subClient) { + try { + await this.subClient.unsubscribe(CONFIG_SYNC_CHANNEL); + await this.subClient.quit(); + } catch { + // Ignore disconnect errors during teardown + } + this.subClient = null; + } + this.isInitialized = false; + this.removeAllListeners(); + } +} + +export const dynamicConfig = DynamicConfigService.getInstance(); diff --git a/src/config/env.config.ts b/src/config/env.config.ts index 9aa5481..3d1062c 100644 --- a/src/config/env.config.ts +++ b/src/config/env.config.ts @@ -1,5 +1,6 @@ import dotenv from 'dotenv'; import { randomUUID } from 'crypto'; +import { dynamicConfig } from './dynamic-config.service.js'; dotenv.config(); @@ -7,6 +8,10 @@ dotenv.config(); const WORKER_ID = process.env.WORKER_ID || `worker-${randomUUID().slice(0, 8)}`; export const env = { + // ======================================================================== + // 1. BOOTSTRAP ENVIRONMENT VARIABLES (from .env / process.env) + // ======================================================================== + // Node Environment NODE_ENV: process.env.NODE_ENV || 'development', @@ -19,48 +24,98 @@ export const env = { // API Server PORT: parseInt(process.env.PORT || "3000"), NS_API_KEY: (process.env.NS_API_KEY || ""), - MAX_BATCH_REQ_LIMIT: parseInt(process.env.MAX_BATCH_REQ_LIMIT || "1000"), DELAYED_PARTITION: parseInt(process.env.DELAYED_PARTITION || "1"), NOTIFICATION_STATUS_PARTITION: parseInt(process.env.NOTIFICATION_STATUS_PARTITION || "1"), // Kafka BROKERS: (process.env.BROKERS?.trim().split(',') || ["localhost:9092"]), - // Background Worker + // Distributed Worker Identity WORKER_ID, - OUTBOX_POLL_INTERVAL_MS: parseInt(process.env.OUTBOX_POLL_INTERVAL_MS || "5000"), - OUTBOX_CLEANUP_INTERVAL_MS: parseInt(process.env.OUTBOX_CLEANUP_INTERVAL_MS || "60000"), - OUTBOX_BATCH_SIZE: parseInt(process.env.OUTBOX_BATCH_SIZE || "100"), - OUTBOX_RETENTION_MS: parseInt(process.env.OUTBOX_RETENTION_MS || "300000"), - OUTBOX_CLAIM_TIMEOUT_MS: parseInt(process.env.OUTBOX_CLAIM_TIMEOUT_MS || "30000"), + + // Logging - Grafana Loki + LOKI_URL: process.env.LOKI_URL || "", + + // Admin Alerts (optional - auto-generated if not provided) + ADMIN_ALERT_ENCRYPTION_KEY: process.env.ADMIN_ALERT_ENCRYPTION_KEY || "", + + // ======================================================================== + // 2. DYNAMIC OPERATIONAL VARIABLES (Delegated to DynamicConfigService) + // These reflect live in-memory MongoDB configuration with zero-restart updates. + // ======================================================================== + + // API + get MAX_BATCH_REQ_LIMIT(): number { + return dynamicConfig?.get?.('api')?.max_batch_req_limit ?? 1000; + }, + + // Background Worker + get OUTBOX_POLL_INTERVAL_MS(): number { + return dynamicConfig?.get?.('worker')?.outbox_poll_interval_ms ?? 5000; + }, + get OUTBOX_CLEANUP_INTERVAL_MS(): number { + return dynamicConfig?.get?.('worker')?.outbox_cleanup_interval_ms ?? 60000; + }, + get OUTBOX_BATCH_SIZE(): number { + return dynamicConfig?.get?.('worker')?.outbox_batch_size ?? 100; + }, + get OUTBOX_RETENTION_MS(): number { + return dynamicConfig?.get?.('worker')?.outbox_retention_ms ?? 300000; + }, + get OUTBOX_CLAIM_TIMEOUT_MS(): number { + return dynamicConfig?.get?.('worker')?.outbox_claim_timeout_ms ?? 30000; + }, // Idempotency & Retry - IDEMPOTENCY_TTL_SECONDS: parseInt(process.env.IDEMPOTENCY_TTL_SECONDS || "86400"), - MAX_RETRY_COUNT: parseInt(process.env.MAX_RETRY_COUNT || "5"), - PROCESSING_TTL_SECONDS: parseInt(process.env.PROCESSING_TTL_SECONDS || "120"), + get IDEMPOTENCY_TTL_SECONDS(): number { + return dynamicConfig?.get?.('retry')?.idempotency_ttl_seconds ?? 86400; + }, + get MAX_RETRY_COUNT(): number { + return dynamicConfig?.get?.('retry')?.max_retry_count ?? 5; + }, + get PROCESSING_TTL_SECONDS(): number { + return dynamicConfig?.get?.('retry')?.processing_ttl_seconds ?? 120; + }, // Delayed Worker - MAX_POLLER_RETRIES: parseInt(process.env.MAX_POLLER_RETRIES || "3"), - DELAYED_POLL_INTERVAL_MS: parseInt(process.env.DELAYED_POLL_INTERVAL_MS || "1000"), - DELAYED_BATCH_SIZE: parseInt(process.env.DELAYED_BATCH_SIZE || "10"), + get MAX_POLLER_RETRIES(): number { + return dynamicConfig?.get?.('delayed')?.max_poller_retries ?? 3; + }, + get DELAYED_POLL_INTERVAL_MS(): number { + return dynamicConfig?.get?.('delayed')?.delayed_poll_interval_ms ?? 1000; + }, + get DELAYED_BATCH_SIZE(): number { + return dynamicConfig?.get?.('delayed')?.delayed_batch_size ?? 10; + }, // Recovery Service - RECOVERY_POLL_INTERVAL_MS: parseInt(process.env.RECOVERY_POLL_INTERVAL_MS || "60000"), - PROCESSING_STUCK_THRESHOLD_MS: parseInt(process.env.PROCESSING_STUCK_THRESHOLD_MS || "300000"), - PENDING_STUCK_THRESHOLD_MS: parseInt(process.env.PENDING_STUCK_THRESHOLD_MS || "300000"), - RECOVERY_BATCH_SIZE: parseInt(process.env.RECOVERY_BATCH_SIZE || "50"), + get RECOVERY_POLL_INTERVAL_MS(): number { + return dynamicConfig?.get?.('recovery')?.recovery_poll_interval_ms ?? 60000; + }, + get PROCESSING_STUCK_THRESHOLD_MS(): number { + return dynamicConfig?.get?.('recovery')?.processing_stuck_threshold_ms ?? 300000; + }, + get PENDING_STUCK_THRESHOLD_MS(): number { + return dynamicConfig?.get?.('recovery')?.pending_stuck_threshold_ms ?? 300000; + }, + get RECOVERY_BATCH_SIZE(): number { + return dynamicConfig?.get?.('recovery')?.recovery_batch_size ?? 50; + }, + get RECOVERY_CLAIM_TIMEOUT_MS(): number { + return dynamicConfig?.get?.('recovery')?.recovery_claim_timeout_ms ?? 60000; + }, + get CLEANUP_RESOLVED_ALERTS_RETENTION_MS(): number { + return dynamicConfig?.get?.('recovery')?.cleanup_resolved_alerts_retention_ms ?? 86400000; + }, + get CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS(): number { + return dynamicConfig?.get?.('recovery')?.cleanup_processed_status_outbox_retention_ms ?? 86400000; + }, - // Cleanup Configuration (run as part of recovery cron) - CLEANUP_RESOLVED_ALERTS_RETENTION_MS: parseInt(process.env.CLEANUP_RESOLVED_ALERTS_RETENTION_MS || "86400000"), // 24 hours - CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS: parseInt(process.env.CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS || "86400000"), // 24 hours - RECOVERY_CLAIM_TIMEOUT_MS: parseInt(process.env.RECOVERY_CLAIM_TIMEOUT_MS || "60000"), // 1 minute - - // Logging - Grafana Loki - LOKI_URL: process.env.LOKI_URL || "", - LOG_LEVEL: process.env.LOG_LEVEL || "info", - LOG_TO_FILE: process.env.LOG_TO_FILE === "true", - - // Admin Alerts (optional - auto-generated if not provided) - // Must be 64-character hex string (32 bytes) if provided - ADMIN_ALERT_ENCRYPTION_KEY: process.env.ADMIN_ALERT_ENCRYPTION_KEY || "", -} \ No newline at end of file + // Logging + get LOG_LEVEL(): string { + return dynamicConfig?.get?.('logging')?.log_level ?? (process.env.LOG_LEVEL || 'info'); + }, + get LOG_TO_FILE(): boolean { + return dynamicConfig?.get?.('logging')?.log_to_file ?? (process.env.LOG_TO_FILE === 'true'); + }, +}; \ No newline at end of file diff --git a/src/database/models/system-config.models.ts b/src/database/models/system-config.models.ts index d4a7093..59418a2 100644 --- a/src/database/models/system-config.models.ts +++ b/src/database/models/system-config.models.ts @@ -15,7 +15,7 @@ const system_config_schema = new mongoose.Schema( index: true, }, value: { - type: String, + type: mongoose.Schema.Types.Mixed, required: true, }, }, @@ -28,7 +28,7 @@ const system_config_schema = new mongoose.Schema( ); const system_config_model = - mongoose.models.SystemConfig || + mongoose.models?.SystemConfig || mongoose.model('SystemConfig', system_config_schema); export default system_config_model; diff --git a/src/processors/delayed/delayed.poller.ts b/src/processors/delayed/delayed.poller.ts index 1908b0a..f205111 100644 --- a/src/processors/delayed/delayed.poller.ts +++ b/src/processors/delayed/delayed.poller.ts @@ -9,6 +9,7 @@ */ import { env } from '@src/config/env.config.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { claimDueEvents, confirmProcessed, reAddToQueue, getDueEventCount, releaseClaim } from './delayed.queue.js'; import { publishToTarget } from './target.producer.js'; import { publishDLQFailureStatus } from './dlq.status.js'; @@ -178,6 +179,15 @@ export const startDelayedPoller = (): void => { logger.success('Delayed poller started'); }; +// Listen for dynamic configuration updates and hot-adjust poller interval +dynamicConfig.on('change', () => { + if (pollerInterval) { + clearInterval(pollerInterval); + pollerInterval = setInterval(pollForDueEvents, env.DELAYED_POLL_INTERVAL_MS); + logger.info(`Adjusted delayed poller interval to ${env.DELAYED_POLL_INTERVAL_MS}ms`); + } +}); + /** * Stop the delayed poller */ diff --git a/src/processors/delayed/delayed.processor.ts b/src/processors/delayed/delayed.processor.ts index 402e554..e065344 100644 --- a/src/processors/delayed/delayed.processor.ts +++ b/src/processors/delayed/delayed.processor.ts @@ -10,6 +10,8 @@ * - Atomic Lua script prevents duplicate processing across multiple workers */ +import { connectMongoDB } from '@src/config/db.config.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { connectRedis, disconnectRedis } from '@src/config/redis.config.js'; import { initTargetProducer, disconnectTargetProducer } from './target.producer.js'; import { initDLQStatusProducer, disconnectDLQStatusProducer } from './dlq.status.js'; @@ -23,6 +25,7 @@ import "@src/admin-alerts/channels/discord.channel.js"; import "@src/admin-alerts/channels/telegram.channel.js"; let isShuttingDown = false; +let dbConnection: Awaited> | null = null; /** * Graceful shutdown handler @@ -56,6 +59,12 @@ const gracefulShutdown = async (signal: string): Promise => { logger.info('Disconnecting Redis...'); await disconnectRedis(); + // 5. Disconnect MongoDB + if (dbConnection) { + logger.info('Disconnecting MongoDB...'); + await dbConnection.disconnect(); + } + logger.success('Graceful shutdown complete'); process.exit(0); } catch (err) { @@ -105,7 +114,13 @@ const main = async (): Promise => { logger.info('Starting Delayed Processor...'); try { - // 1. Connect to Redis (for delayed queue) + // 1. Connect to MongoDB & Initialize Dynamic Configuration + logger.info('Connecting to MongoDB...'); + dbConnection = await connectMongoDB(); + logger.success('Connected to MongoDB'); + await dynamicConfig.initialize(false); + + // 2. Connect to Redis (for delayed queue) logger.info('Connecting to Redis...'); await connectRedis(); diff --git a/src/processors/unified/unified.processor.ts b/src/processors/unified/unified.processor.ts index 97a0e01..46549e5 100644 --- a/src/processors/unified/unified.processor.ts +++ b/src/processors/unified/unified.processor.ts @@ -8,6 +8,7 @@ */ import { connectMongoDB } from '@src/config/db.config.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { connectRedis, disconnectRedis } from '@src/config/redis.config.js'; import { initStatusProducer, disconnectStatusProducer } from '@src/processors/shared/status.producer.js'; import { initDelayedProducer, disconnectDelayedProducer } from '@src/processors/shared/delayed.producer.js'; @@ -191,6 +192,9 @@ const main = async (): Promise => { dbConnection = await connectMongoDB(); logger.success('Successfully connected to MongoDB'); + // Initialize Dynamic Configuration + await dynamicConfig.initialize(false); + // 2. Connect to Redis logger.info('Connecting to Redis...'); await connectRedis(); diff --git a/src/types/schemas.ts b/src/types/schemas.ts index 92b637c..71970ce 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -15,7 +15,6 @@ import { } from "./types.js"; import type { UUID } from "crypto"; import { validate, version } from "uuid"; -import { env } from "@src/config/env.config.js"; // ============================================================================ // BASE FIELD SCHEMAS @@ -409,22 +408,22 @@ export const baseBatchNotificationRequestSchema = z }), ).min(1, "At least one recipient is required."), }) - .refine( - (data) => { - // limit check - if ( - data.recipients && - data.recipients.length * data.channel.length > env.MAX_BATCH_REQ_LIMIT - ) { - return false; - } - return true; - }, - { - message: `Batch size exceeds limit (${env.MAX_BATCH_REQ_LIMIT})`, - path: ["recipients"], - }, - ) + .superRefine((data, ctx) => { + const maxLimit = + typeof process !== "undefined" && process.env.MAX_BATCH_REQ_LIMIT + ? parseInt(process.env.MAX_BATCH_REQ_LIMIT) + : 1000; + if ( + data.recipients && + data.recipients.length * data.channel.length > maxLimit + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Batch size exceeds limit (${maxLimit})`, + path: ["recipients"], + }); + } + }) .superRefine((data, ctx) => { if ( !hasNonEmptyTemplateIds(data.template_id) && @@ -495,3 +494,63 @@ export const safeValidateNotificationTemplateRequestSchema = (data: unknown) => notificationTemplateRequestSchema.safeParse(data); export const safeValidateNotificationTemplateUpdateRequestSchema = (data: unknown) => notificationTemplateUpdateRequestSchema.safeParse(data); + +// ============================================================================ +// DYNAMIC OPERATIONAL SETTINGS & ADMIN AUTH SCHEMAS +// ============================================================================ + +export const operationalSettingsSchema = z.object({ + api: z.object({ + max_batch_req_limit: z.number().int().min(10).max(10000).default(1000), + }), + worker: z.object({ + outbox_poll_interval_ms: z.number().int().min(500).max(60000).default(5000), + outbox_cleanup_interval_ms: z.number().int().min(5000).max(600000).default(60000), + outbox_batch_size: z.number().int().min(1).max(2000).default(100), + outbox_retention_ms: z.number().int().min(10000).max(86400000).default(300000), + outbox_claim_timeout_ms: z.number().int().min(5000).max(300000).default(30000), + }), + retry: z.object({ + max_retry_count: z.number().int().min(1).max(30).default(5), + idempotency_ttl_seconds: z.number().int().min(60).max(2592000).default(86400), + processing_ttl_seconds: z.number().int().min(10).max(1800).default(120), + }), + delayed: z.object({ + delayed_poll_interval_ms: z.number().int().min(200).max(30000).default(1000), + delayed_batch_size: z.number().int().min(1).max(500).default(10), + max_poller_retries: z.number().int().min(1).max(10).default(3), + }), + recovery: z.object({ + recovery_poll_interval_ms: z.number().int().min(5000).max(600000).default(60000), + processing_stuck_threshold_ms: z.number().int().min(10000).max(3600000).default(300000), + pending_stuck_threshold_ms: z.number().int().min(10000).max(3600000).default(300000), + recovery_batch_size: z.number().int().min(1).max(500).default(50), + recovery_claim_timeout_ms: z.number().int().min(5000).max(300000).default(60000), + cleanup_resolved_alerts_retention_ms: z.number().int().min(60000).max(2592000000).default(86400000), + cleanup_processed_status_outbox_retention_ms: z.number().int().min(60000).max(2592000000).default(86400000), + }), + logging: z.object({ + log_level: z.enum(['debug', 'info', 'warn', 'error']).default('info'), + log_to_file: z.boolean().default(true), + }), +}); + +export const partialOperationalSettingsSchema = z.object({ + api: operationalSettingsSchema.shape.api.partial().optional(), + worker: operationalSettingsSchema.shape.worker.partial().optional(), + retry: operationalSettingsSchema.shape.retry.partial().optional(), + delayed: operationalSettingsSchema.shape.delayed.partial().optional(), + recovery: operationalSettingsSchema.shape.recovery.partial().optional(), + logging: operationalSettingsSchema.shape.logging.partial().optional(), +}); + +export const adminSetupSchema = z.object({ + username: z.string().trim().min(3, "Username must be at least 3 characters").max(50, "Username cannot exceed 50 characters"), + password: z.string().min(8, "Password must be at least 8 characters").max(64, "Password cannot exceed 64 characters"), +}); + +export const adminLoginSchema = z.object({ + username: z.string().trim().min(1, "Username is required"), + password: z.string().min(1, "Password is required"), +}); + diff --git a/src/types/types.ts b/src/types/types.ts index 4cfc588..0bb1594 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -128,6 +128,10 @@ import { pluginSchema, providerSchema, channelRoutingSchema, + operationalSettingsSchema, + partialOperationalSettingsSchema, + adminSetupSchema, + adminLoginSchema, } from "./schemas.js"; // ============================================================================ @@ -407,7 +411,8 @@ export type ServiceContext = | 'adminAlert' | 'pluginLoader' | 'pluginRegistry' - | 'rateLimiter'; + | 'rateLimiter' + | 'configSync'; export interface LogMeta { notificationId?: string; @@ -446,3 +451,36 @@ export interface ValidatedOutboxEntry { payload: Record; status: string; } + +// ============================================================================ +// DYNAMIC OPERATIONAL SETTINGS & ADMIN AUTH TYPES +// ============================================================================ + +export type OperationalSettings = z.infer; +export type PartialOperationalSettings = z.infer; +export type OperationalSettingsGroup = keyof OperationalSettings; + +export type SystemConfigSyncAction = 'SETTINGS_SEEDED' | 'SETTINGS_UPDATED' | 'SETTINGS_RESET'; + +export interface SystemConfigSyncMessage { + action: SystemConfigSyncAction; + source_instance_id: string; + timestamp: string; + payload?: PartialOperationalSettings; +} + +export interface AdminCredentialsDoc { + username: string; + password_hash: string; + salt: string; + created_at: string; + updated_at?: string; +} + +export type AdminSetupPayload = z.infer; +export type AdminLoginPayload = z.infer; + +export interface AdminAuthStatus { + isConfigured: boolean; +} + diff --git a/src/workers/consumers/status.consumer.ts b/src/workers/consumers/status.consumer.ts index e45899e..e408199 100644 --- a/src/workers/consumers/status.consumer.ts +++ b/src/workers/consumers/status.consumer.ts @@ -121,7 +121,7 @@ const processStatusMessage = async ({ partition, message }: EachMessagePayload): const result = await notification_model.findByIdAndUpdate( data.notification_id, updateData, - { new: true } + { returnDocument: 'after' } ); if (result) { diff --git a/src/workers/cron/background.cron.ts b/src/workers/cron/background.cron.ts index fd7429e..4b27a34 100644 --- a/src/workers/cron/background.cron.ts +++ b/src/workers/cron/background.cron.ts @@ -1,4 +1,5 @@ import { env } from "@src/config/env.config.js"; +import { dynamicConfig } from "@src/config/dynamic-config.service.js"; import { OUTBOX_STATUS, type OutboxCronState } from "@src/types/types.js"; import outbox_model from "@src/database/models/outbox.models.js"; import status_outbox_model from "@src/database/models/status-outbox.models.js"; @@ -51,7 +52,7 @@ const claimOutboxEntries = async (): Promise => { } }, { - new: true, + returnDocument: 'after', sort: { created_at: 1 } // FIFO order } ); @@ -146,7 +147,7 @@ const claimStatusOutboxEntries = async (): Promise => { } }, { - new: true, + returnDocument: 'after', sort: { created_at: 1 } } ); @@ -210,6 +211,29 @@ export const startCronJobs = (): void => { logger.success("Cron jobs started"); }; +// Listen for dynamic configuration updates and hot-adjust intervals +dynamicConfig.on('change', () => { + if (state.shouldStop) return; + + if (state.pollIntervalId) { + clearInterval(state.pollIntervalId); + state.pollIntervalId = setInterval(pollOutbox, env.OUTBOX_POLL_INTERVAL_MS); + logger.info(`Adjusted outbox poll interval to ${env.OUTBOX_POLL_INTERVAL_MS}ms`); + } + + if (state.statusPollIntervalId) { + clearInterval(state.statusPollIntervalId); + state.statusPollIntervalId = setInterval(pollStatusOutbox, env.OUTBOX_POLL_INTERVAL_MS); + logger.info(`Adjusted status outbox poll interval to ${env.OUTBOX_POLL_INTERVAL_MS}ms`); + } + + if (state.cleanupIntervalId) { + clearInterval(state.cleanupIntervalId); + state.cleanupIntervalId = setInterval(cleanupPublishedEvents, env.OUTBOX_CLEANUP_INTERVAL_MS); + logger.info(`Adjusted outbox cleanup interval to ${env.OUTBOX_CLEANUP_INTERVAL_MS}ms`); + } +}); + /** * Stop the cron jobs gracefully */ diff --git a/src/workers/recovery/recovery.cron.ts b/src/workers/recovery/recovery.cron.ts index 7751847..547865d 100644 --- a/src/workers/recovery/recovery.cron.ts +++ b/src/workers/recovery/recovery.cron.ts @@ -13,6 +13,7 @@ import mongoose from 'mongoose'; import { env } from '@src/config/env.config.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { NOTIFICATION_STATUS, ALERT_TYPE, type HealthChecker, type RecoveryCronState } from '@src/types/types.js'; import notification_model from '@src/database/models/notification.models.js'; import alert_model from '@src/database/models/alert.models.js'; @@ -61,7 +62,7 @@ const recoverStuckProcessing = async (): Promise => { recovery_claimed_at: now } }, - { new: true } + { returnDocument: 'after' } ); if (!notification) break; @@ -291,7 +292,7 @@ const detectOrphanedPending = async (): Promise => { recovery_claimed_at: now } }, - { new: true } + { returnDocument: 'after' } ); if (!notification) break; @@ -403,7 +404,7 @@ const autoResolveDeliveredAlerts = async (): Promise => { recovery_claimed_at: now } }, - { new: true } + { returnDocument: 'after' } ); if (!alert) break; @@ -526,6 +527,16 @@ export const startRecoveryCron = (): void => { logger.success('Recovery cron started'); }; +// Listen for dynamic configuration updates and hot-adjust recovery interval +dynamicConfig.on('change', () => { + if (state.shouldStop) return; + if (state.intervalId) { + clearInterval(state.intervalId); + state.intervalId = setInterval(runRecovery, env.RECOVERY_POLL_INTERVAL_MS); + logger.info(`Adjusted recovery cron interval to ${env.RECOVERY_POLL_INTERVAL_MS}ms`); + } +}); + /** * Stop the recovery cron job gracefully */ diff --git a/src/workers/recovery/recovery.service.ts b/src/workers/recovery/recovery.service.ts index 6d8a8b4..1ff2643 100644 --- a/src/workers/recovery/recovery.service.ts +++ b/src/workers/recovery/recovery.service.ts @@ -14,6 +14,7 @@ import mongoose from 'mongoose'; import { env } from '@src/config/env.config.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { connectRedis, disconnectRedis, getRedisClient } from '@src/config/redis.config.js'; import { startRecoveryCron, stopRecoveryCron, setHealthChecker } from './recovery.cron.js'; import { recoveryLogger as logger, flushLogs } from '@src/workers/utils/logger.js'; @@ -264,6 +265,10 @@ const main = async (): Promise => { logger.info('Recovery service will start anyway and retry connections...'); } + if (mongoConnected) { + await dynamicConfig.initialize(false); + } + // Set health checker for recovery cron setHealthChecker(checkHealth); diff --git a/src/workers/utils/logger.ts b/src/workers/utils/logger.ts index 349640a..429a9d9 100644 --- a/src/workers/utils/logger.ts +++ b/src/workers/utils/logger.ts @@ -5,7 +5,6 @@ import winston from 'winston'; import LokiTransport from 'winston-loki'; -import { env } from '@src/config/env.config.js'; import type { LogMeta, Logger, ServiceContext } from '@src/types/types.js'; // Service context types @@ -22,7 +21,8 @@ const SERVICE_LABELS = { adminAlert: 'admin-alert', pluginLoader: 'plugin-loader', pluginRegistry: 'plugin-registry', - rateLimiter: 'rate-limiter' + rateLimiter: 'rate-limiter', + configSync: 'config-sync' } as const; // Custom log levels including 'success' @@ -68,7 +68,8 @@ const SERVICE_EMOJI: Record = { adminAlert: '🔔', pluginLoader: '🔌', pluginRegistry: '📦', - rateLimiter: '⏱️' + rateLimiter: '⏱️', + configSync: '⚙️' }; /** @@ -111,15 +112,15 @@ const createWinstonLogger = (service: ServiceContext) => { ]; // Add Loki transport if LOKI_URL is configured - if (env.LOKI_URL) { + if (process.env.LOKI_URL) { transports.push( new LokiTransport({ - host: env.LOKI_URL, + host: process.env.LOKI_URL, labels: { job: 'notification-service', service: SERVICE_LABELS[service], - workerId: env.WORKER_ID, - environment: env.NODE_ENV + workerId: process.env.WORKER_ID || 'worker-default', + environment: process.env.NODE_ENV || 'development' }, json: true, batching: true, @@ -133,7 +134,9 @@ const createWinstonLogger = (service: ServiceContext) => { } // Add file transport in production - if (env.NODE_ENV === 'production' && env.LOG_TO_FILE) { + const isProd = (process.env.NODE_ENV || 'development') === 'production'; + const logToFile = process.env.LOG_TO_FILE === 'true'; + if (isProd && logToFile) { transports.push( new winston.transports.File({ filename: `logs/${SERVICE_LABELS[service]}.log`, @@ -151,15 +154,29 @@ const createWinstonLogger = (service: ServiceContext) => { ); } - return winston.createLogger({ + const winstonLogger = winston.createLogger({ levels: customLevels.levels, - level: env.LOG_LEVEL || 'info', + level: process.env.LOG_LEVEL || 'info', defaultMeta: { service: SERVICE_LABELS[service], - workerId: "worker-default" + workerId: process.env.WORKER_ID || "worker-default" }, transports }); + + activeWinstonLoggers.push(winstonLogger); + return winstonLogger; +}; + +const activeWinstonLoggers: winston.Logger[] = []; + +/** + * Dynamically update log level across all active loggers + */ +export const setLogLevel = (newLevel: string): void => { + for (const l of activeWinstonLoggers) { + l.level = newLevel; + } }; /** @@ -207,6 +224,7 @@ export const pluginLoaderLogger = createLogger('pluginLoader'); export const pluginRegistryLogger = createLogger('pluginRegistry'); export const rateLimiterLogger = createLogger('rateLimiter'); export const adminAlertLogger = createLogger('adminAlert'); +export const configSyncLogger = createLogger('configSync'); /** * Graceful shutdown - flush all logs before exit diff --git a/src/workers/worker.ts b/src/workers/worker.ts index 1a4fa8e..1ccc129 100644 --- a/src/workers/worker.ts +++ b/src/workers/worker.ts @@ -1,4 +1,5 @@ import { connectMongoDB } from "@src/config/db.config.js"; +import { dynamicConfig } from "@src/config/dynamic-config.service.js"; import { initProducer, disconnectProducer } from "@src/workers/producers/background.producer.js"; import { startCronJobs, stopCronJobs } from "@src/workers/cron/background.cron.js"; import { startStatusConsumer, stopStatusConsumer } from "@src/workers/consumers/status.consumer.js"; @@ -97,6 +98,9 @@ const main = async (): Promise => { dbConnection = await connectMongoDB(); logger.success("MongoDB connected"); + // Initialize Dynamic Configuration + await dynamicConfig.initialize(false); + // 2. Initialize Kafka producer logger.info("Initializing Kafka producer..."); await initProducer(); diff --git a/tests/integration/api/admin-auth.test.ts b/tests/integration/api/admin-auth.test.ts new file mode 100644 index 0000000..7187e26 --- /dev/null +++ b/tests/integration/api/admin-auth.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import admin_auth_router from '@src/api/routes/admin-auth.routes.js'; +import system_config_model from '@src/database/models/system-config.models.js'; + +vi.mock('@src/database/models/system-config.models.js', () => ({ + default: { + findOne: vi.fn(), + create: vi.fn(), + findOneAndUpdate: vi.fn(), + }, +})); + +describe('Admin Auth Routes Integration', () => { + let app: express.Application; + + beforeEach(() => { + vi.clearAllMocks(); + app = express(); + app.use(express.json()); + app.use('/api/admin/auth', admin_auth_router); + }); + + describe('GET /api/admin/auth/status', () => { + it('should return isConfigured: false when no credentials exist in database', async () => { + vi.mocked(system_config_model.findOne).mockResolvedValue(null); + + const res = await request(app).get('/api/admin/auth/status'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ isConfigured: false }); + }); + + it('should return isConfigured: true when credentials exist in database', async () => { + vi.mocked(system_config_model.findOne).mockResolvedValue({ + key: 'admin_credentials', + value: { username: 'admin' }, + } as any); + + const res = await request(app).get('/api/admin/auth/status'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ isConfigured: true }); + }); + }); + + describe('POST /api/admin/auth/setup', () => { + it('should reject setup with validation error if password too short', async () => { + const res = await request(app) + .post('/api/admin/auth/setup') + .send({ username: 'admin', password: '123' }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Validation failed'); + }); + + it('should reject setup with validation error if password exceeds max length', async () => { + const res = await request(app) + .post('/api/admin/auth/setup') + .send({ username: 'admin', password: 'a'.repeat(65) }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Validation failed'); + }); + + it('should reject setup if already configured', async () => { + vi.mocked(system_config_model.findOne).mockResolvedValue({ + key: 'admin_credentials', + value: { username: 'admin' }, + } as any); + + const res = await request(app) + .post('/api/admin/auth/setup') + .send({ username: 'admin', password: 'supersecretpassword123' }); + + expect(res.status).toBe(400); + expect(res.body.error).toContain('already been configured'); + }); + + it('should successfully configure admin and return 201', async () => { + vi.mocked(system_config_model.findOne).mockResolvedValue(null); + vi.mocked(system_config_model.create).mockResolvedValue({} as any); + + const res = await request(app) + .post('/api/admin/auth/setup') + .send({ username: 'superuser', password: 'supersecretpassword123' }); + + expect(res.status).toBe(201); + expect(res.body.success).toBe(true); + expect(res.body.user.username).toBe('superuser'); + expect(system_config_model.create).toHaveBeenCalled(); + }); + }); + + describe('POST /api/admin/auth/verify', () => { + it('should reject invalid credentials', async () => { + vi.mocked(system_config_model.findOne).mockResolvedValue(null); + + const res = await request(app) + .post('/api/admin/auth/verify') + .send({ username: 'wrong', password: 'wrongpassword' }); + + expect(res.status).toBe(401); + expect(res.body.isValid).toBe(false); + }); + }); +}); diff --git a/tests/integration/api/settings.test.ts b/tests/integration/api/settings.test.ts new file mode 100644 index 0000000..ba5808d --- /dev/null +++ b/tests/integration/api/settings.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import settings_router from '@src/api/routes/settings.routes.js'; +import { auth_middleware } from '@src/api/middlewares/auth_middleware.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; + +vi.mock('@src/config/redis.config.js', () => ({ + getRedisClient: vi.fn(() => ({ + publish: vi.fn().mockResolvedValue(1), + duplicate: vi.fn(() => ({ + status: 'ready', + connect: vi.fn().mockResolvedValue(undefined), + subscribe: vi.fn().mockResolvedValue(undefined), + unsubscribe: vi.fn().mockResolvedValue(undefined), + quit: vi.fn().mockResolvedValue('OK'), + on: vi.fn(), + })), + })), +})); + +vi.mock('@src/database/models/system-config.models.js', () => ({ + default: { + findOne: vi.fn(), + findOneAndUpdate: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + }, +})); + +describe('Settings API Integration', () => { + let app: express.Application; + + beforeEach(() => { + vi.clearAllMocks(); + app = express(); + app.use(express.json()); + app.use('/api/settings', auth_middleware, settings_router); + }); + + it('should reject unauthorized requests when bearer token is missing', async () => { + const res = await request(app).get('/api/settings'); + expect(res.status).toBe(401); + }); + + it('should return operational settings with valid authorization', async () => { + const res = await request(app) + .get('/api/settings') + .set('Authorization', `Bearer ${process.env.NS_API_KEY}`); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.settings.worker.outbox_batch_size).toBeDefined(); + }); + + it('should update operational settings with valid authorization', async () => { + const res = await request(app) + .put('/api/settings') + .set('Authorization', `Bearer ${process.env.NS_API_KEY}`) + .send({ + worker: { + outbox_batch_size: 150, + }, + logging: { + log_level: 'warn', + }, + }); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.settings.worker.outbox_batch_size).toBe(150); + expect(res.body.settings.logging.log_level).toBe('warn'); + }); + + it('should reject update with invalid schema bounds', async () => { + const res = await request(app) + .put('/api/settings') + .set('Authorization', `Bearer ${process.env.NS_API_KEY}`) + .send({ + worker: { + outbox_batch_size: 99999, // exceeds max 2000 + }, + }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Validation failed'); + }); + + it('should reset operational settings to system defaults', async () => { + const res = await request(app) + .post('/api/settings/reset') + .set('Authorization', `Bearer ${process.env.NS_API_KEY}`); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.settings.worker.outbox_batch_size).toBe(100); + }); +}); diff --git a/tests/unit/config/dynamic-config.test.ts b/tests/unit/config/dynamic-config.test.ts new file mode 100644 index 0000000..45b47f4 --- /dev/null +++ b/tests/unit/config/dynamic-config.test.ts @@ -0,0 +1,135 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { + operationalSettingsSchema, + partialOperationalSettingsSchema, + adminSetupSchema, + adminLoginSchema, +} from '@src/types/schemas.js'; +import { + dynamicConfig, + SYSTEM_DEFAULT_OPERATIONAL_SETTINGS, +} from '@src/config/dynamic-config.service.js'; +import { env } from '@src/config/env.config.js'; + +vi.mock('@src/config/redis.config.js', () => ({ + getRedisClient: vi.fn(() => ({ + publish: vi.fn().mockResolvedValue(1), + duplicate: vi.fn(() => ({ + status: 'ready', + connect: vi.fn().mockResolvedValue(undefined), + subscribe: vi.fn().mockResolvedValue(undefined), + unsubscribe: vi.fn().mockResolvedValue(undefined), + quit: vi.fn().mockResolvedValue('OK'), + on: vi.fn(), + })), + })), +})); + +vi.mock('@src/database/models/system-config.models.js', () => ({ + default: { + findOne: vi.fn(), + findOneAndUpdate: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + }, +})); + +describe('Dynamic Operational Configuration', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe('Schema Validations', () => { + it('should validate SYSTEM_DEFAULT_OPERATIONAL_SETTINGS against schema', () => { + const parsed = operationalSettingsSchema.safeParse(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS); + expect(parsed.success).toBe(true); + }); + + it('should reject out-of-bound batch sizes', () => { + const invalid = { + ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS, + worker: { + ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS.worker, + outbox_batch_size: 0, // min is 1 + }, + }; + const parsed = operationalSettingsSchema.safeParse(invalid); + expect(parsed.success).toBe(false); + }); + + it('should reject invalid log levels', () => { + const invalid = { + ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS, + logging: { + log_level: 'verbose', // invalid enum + log_to_file: true, + }, + }; + const parsed = operationalSettingsSchema.safeParse(invalid); + expect(parsed.success).toBe(false); + }); + + it('should allow valid partial operational settings', () => { + const partial = { + worker: { + outbox_batch_size: 250, + }, + logging: { + log_level: 'debug' as const, + }, + }; + const parsed = partialOperationalSettingsSchema.safeParse(partial); + expect(parsed.success).toBe(true); + expect(parsed.data?.worker?.outbox_batch_size).toBe(250); + }); + + it('should validate admin setup requirements', () => { + expect(adminSetupSchema.safeParse({ username: 'ad', password: '123' }).success).toBe(false); + expect(adminSetupSchema.safeParse({ username: 'admin', password: 'short' }).success).toBe(false); + expect(adminSetupSchema.safeParse({ username: 'admin', password: 'securepassword123' }).success).toBe(true); + }); + + it('should validate admin login payload', () => { + expect(adminLoginSchema.safeParse({ username: '', password: '' }).success).toBe(false); + expect(adminLoginSchema.safeParse({ username: 'admin', password: 'password123' }).success).toBe(true); + }); + }); + + describe('DynamicConfigService and Env Getters', () => { + it('should return default settings initially', () => { + const all = dynamicConfig.getAll(); + expect(all.worker.outbox_batch_size).toBeDefined(); + expect(dynamicConfig.get('worker').outbox_batch_size).toBe(all.worker.outbox_batch_size); + }); + + it('should dynamically update settings and reflect via env getters', async () => { + let changeFired = false; + const listener = () => { + changeFired = true; + }; + dynamicConfig.on('change', listener); + + await dynamicConfig.updateSettings({ + worker: { + outbox_batch_size: 42, + }, + logging: { + log_level: 'debug', + }, + }); + + expect(dynamicConfig.get('worker').outbox_batch_size).toBe(42); + expect(env.OUTBOX_BATCH_SIZE).toBe(42); + expect(env.LOG_LEVEL).toBe('debug'); + expect(changeFired).toBe(true); + + dynamicConfig.off('change', listener); + }); + + it('should reset settings to system defaults', async () => { + await dynamicConfig.resetSettings(); + expect(dynamicConfig.get('worker').outbox_batch_size).toBe(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS.worker.outbox_batch_size); + expect(env.OUTBOX_BATCH_SIZE).toBe(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS.worker.outbox_batch_size); + expect(env.LOG_LEVEL).toBe(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS.logging.log_level); + }); + }); +}); From 9d4bc5dbc9e2a14179d2f08d3af8515c73d4acd8 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Fri, 18 Sep 2026 18:24:20 +0530 Subject: [PATCH 15/46] feat: Add initial settings state and unsaved changes indicator in SettingsPage --- dashboard/app/settings/page.tsx | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/dashboard/app/settings/page.tsx b/dashboard/app/settings/page.tsx index 48e4eb5..3f54ed0 100644 --- a/dashboard/app/settings/page.tsx +++ b/dashboard/app/settings/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { useEffect, useState } from "react"; +import { useEffect, useState, useMemo } from "react"; import { DashboardLayout } from "@/components/layout/dashboard-layout"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Separator } from "@/components/ui/separator"; @@ -36,11 +36,18 @@ export default function SettingsPage() { const { restartTour } = useTour(); const [settings, setSettings] = useState(null); + const [initialSettings, setInitialSettings] = useState(null); const [isLoading, setIsLoading] = useState(true); const [isSaving, setIsSaving] = useState(false); const [isResetting, setIsResetting] = useState(false); const [showAdvanced, setShowAdvanced] = useState(false); + // Compute whether operational settings have been modified from original loaded state + const hasChanges = useMemo(() => { + if (!settings || !initialSettings) return false; + return JSON.stringify(settings) !== JSON.stringify(initialSettings); + }, [settings, initialSettings]); + // Fetch operational settings on mount useEffect(() => { const fetchSettings = async () => { @@ -48,6 +55,7 @@ export default function SettingsPage() { const res = await settingsService.get(); if (res.success && res.settings) { setSettings(res.settings); + setInitialSettings(res.settings); } } catch (err) { console.error("Failed to load settings", err); @@ -67,6 +75,7 @@ export default function SettingsPage() { const res = await settingsService.update(settings); if (res.success && res.settings) { setSettings(res.settings); + setInitialSettings(res.settings); toast.success("Settings updated and synchronized across all instances!"); } } catch (err: unknown) { @@ -87,6 +96,7 @@ export default function SettingsPage() { const res = await settingsService.reset(); if (res.success && res.settings) { setSettings(res.settings); + setInitialSettings(res.settings); toast.success("Operational settings reset to system defaults!"); } } catch (err: unknown) { @@ -237,7 +247,14 @@ export default function SettingsPage() { {/* Action Bar inside Advanced Settings */}
-

Live Engine Tuning

+
+

Live Engine Tuning

+ {hasChanges && ( + + Unsaved Changes + + )} +

Changes synchronize immediately across all running containers without service restarts.

@@ -260,7 +277,7 @@ export default function SettingsPage() { + ); + })} +
+ + {/* Step Highlight Preview Box */} +
+
+ {currentStep === 1 && } + {currentStep === 2 && } + {currentStep === 3 && } + {currentStep === 4 && } + + {currentStep === 1 && "Orchestration, Not Delivery"} + {currentStep === 2 && "Modular, Scalable & Extensible"} + {currentStep === 3 && "Secure Master Account"} + {currentStep === 4 && "Optional Provider Connectors"} +
+

+ {currentStep === 1 && + "SimpleNS manages the complex notification lifecycle (retries, rate limiting, and recovery) while delegating actual sending to lightweight plugins."} + {currentStep === 2 && + "Enjoy complete data sovereignty, AI integration via Model Context Protocol (MCP), independent channel scaling, and zero per-notification vendor fees."} + {currentStep === 3 && + "Master administrator credentials are cryptographically protected and stored securely in MongoDB for complete operational authority."} + {currentStep === 4 && + "Install delivery provider plugins now or explore the full catalog at any time inside the dashboard."} +

@@ -196,185 +464,736 @@ export default function SetupPage() {
- {/* Right Column: Clean Elegant Setup Form */} -
- - {/* Mobile Logo */} -
- SimpleNS Logo -
- Initial Setup -
+ {/* Right Column: Step Content Area */} +
+ {/* Mobile Top Header */} +
+ SimpleNS Logo +
+ Step {currentStep} of 4
+
- {/* Header */} -
-

- Create Administrator -

-

- Set up your master credentials to secure and manage this SimpleNS instance. -

-
+
+ + {/* ================= STEP 1: WHAT IS SIMPLENS ================= */} + {currentStep === 1 && ( + +
+
+ + Step 1 of 4 • Introduction +
+

+ What is SimpleNS? +

+

+ SimpleNS (Simple Notification System) is a self-hosted notification orchestration engine that solves your notification infrastructure once and for all. +

+
+ + {/* Core Concept Callout */} +
+
+ + Core Philosophy: Orchestration, Not Delivery +
+

+ SimpleNS handles the complex, heavy-lifting infrastructure logic—exponential backoff retries, rate limiting, delayed queues, idempotency deduplication, and crash recovery—while delegating actual delivery to lightweight, swappable plugins. +

+
+ + {/* 3 Essential Pillars */} +
+
+
+ +
+

+ 100% Self-Hosted +

+

+ Runs on your infrastructure. Maintain total data sovereignty and customer privacy with zero per-notification vendor fees. +

+
+ +
+
+ +
+

+ Plugin-Based +

+

+ Swap delivery providers (Email, SMS, Push, Chat) at any time without modifying a single line of application code. +

+
- {/* Form */} -
- {error && ( -
- - {error} -
+
+
+ +
+

+ Zero Lock-In +

+

+ Standard REST APIs, webhook status callbacks, and open-source architecture that puts you in full control. +

+
+
+ + {/* Step 1 Navigation */} +
+ +
+ )} - {/* Username */} -
- - setUsername(e.target.value)} - required - minLength={3} - maxLength={50} - disabled={isLoading} - className="h-11 bg-background text-sm font-medium" - /> -

- Between 3 and 50 characters. -

-
+ {/* ================= STEP 2: WHAT IT CAN DO ================= */} + {currentStep === 2 && ( + +
+
+ + Step 2 of 4 • Core Features +
+

+ What Can SimpleNS Do? +

+

+ A powerful, developer-friendly toolkit engineered for reliable notifications, effortless scaling, and modern AI automation. +

+
- {/* Password */} -
- -
- setPassword(e.target.value)} - required - minLength={8} - maxLength={64} - disabled={isLoading} - className="h-11 bg-background text-sm font-medium pr-10" - /> - -
- - {/* Minimalist Password Strength Bar */} - {password.length > 0 && ( -
-
- Password strength - - {strengthLabel} - + {/* 6 High-Level Features Grid */} +
+ {/* Feature 1: Extensibility */} +
+
+ +
+
+

Modular Extensibility

+

+ Install official or community plugins for Email (SMTP/SES/Resend), SMS (Twilio/SNS), Push, Discord, Telegram, and Webhooks. +

+
- {/* Continuous sleek bar */} -
-
+ + {/* Feature 2: MCP Server & AI */} +
+
+ +
+
+
+

AI & MCP Server Support

+ Native +
+

+ Built-in Model Context Protocol (MCP) server allows AI agents (Claude, Gemini, Cursor) to send notifications and resolve alerts autonomously. +

+
+
+ + {/* Feature 3: Scalability */} +
+
+ +
+
+

Horizontal Scalability

+

+ Scale notification workers independently per channel. Effortlessly absorb millions of deliveries with zero bottlenecks. +

+
+
+ + {/* Feature 4: Self-Hosting */} +
+
+ +
+
+

Self-Hosted & Private

+

+ Deploy effortlessly with Docker Compose or Kubernetes. All sensitive customer credentials and logs stay in your private network. +

+
+
+ + {/* Feature 5: Scheduled & Delayed Queueing */} +
+
+ +
+
+

Scheduled & Delayed Queue

+

+ Queue notifications for future campaigns, set reminders, or delay dispatch with millisecond precision and automatic retry fallbacks. +

+
+
+ + {/* Feature 6: Built-in Resilience */} +
+
+ +
+
+

Resilience & Recovery

+

+ Automated exponential backoff retries, provider rate limiting, ghost detection, and 1-click admin retry resolution. +

+
- )} -

- 8 to 64 characters with a mix of letters and numbers. -

-
+ {/* Step 2 Navigation */} +
+ + +
+ + )} - {/* Confirm Password */} -
- -
- setConfirmPassword(e.target.value)} - required - minLength={8} - maxLength={64} - disabled={isLoading} - className="h-11 bg-background text-sm font-medium pr-10" - /> - -
- {confirmPassword.length > 0 && ( -

- {passwordsMatch ? ( - <> - Passwords match - - ) : ( - "Passwords do not match" + {/* ================= STEP 3: ADMIN SETUP ================= */} + {currentStep === 3 && ( + +

+
+ + Step 3 of 4 • Master Account +
+

+ Create Administrator +

+

+ Configure your master administrator credentials to secure and manage this SimpleNS instance. +

+
+ + {/* Setup Form */} + + {error && ( +
+ + {error} +
)} -

- )} -
- {/* Submit Button */} - - - + {/* Username */} +
+ + setUsername(e.target.value)} + required + minLength={3} + maxLength={50} + disabled={isLoading} + className="h-11 bg-background text-sm font-medium" + /> +

+ Between 3 and 50 characters. +

+
+ + {/* Password */} +
+ +
+ setPassword(e.target.value)} + required + minLength={8} + maxLength={64} + disabled={isLoading} + className="h-11 bg-background text-sm font-medium pr-10" + /> + +
+ + {/* Minimalist Password Strength Bar */} + {password.length > 0 && ( +
+
+ Password strength + + {strengthLabel} + +
+
+
+
+
+ )} + +

+ 8 to 64 characters with letters, numbers, and symbols. +

+
+ + {/* Confirm Password */} +
+ +
+ setConfirmPassword(e.target.value)} + required + minLength={8} + maxLength={64} + disabled={isLoading} + className="h-11 bg-background text-sm font-medium pr-10" + /> + +
+ {confirmPassword.length > 0 && ( +

+ {passwordsMatch ? ( + <> + Passwords match + + ) : ( + "Passwords do not match" + )} +

+ )} +
+ + {/* Step 3 Navigation Buttons */} +
+ + + +
+ + + )} + + {/* ================= STEP 4: OPTIONAL PLUGINS & PROVIDERS ================= */} + {currentStep === 4 && ( + + {/* Top Banner with Skip button */} +
+
+ + Admin Created • Optional Setup +
+ +
+ + {/* ================= PHASE 4A: DISCOVER & INSTALL ================= */} + {pluginPhase === "install" && ( +
+
+

+ Install Notification Plugins +

+

+ SimpleNS uses plugins to deliver messages to real-world services. Install one or more official plugins to get started, or skip and install them later in the dashboard. +

+
+ + {isCatalogLoading ? ( +
+ + Loading plugin catalog... +
+ ) : ( +
+ {catalog.map((item) => { + const isInstalled = installedPackages.has(item.package); + const isInstalling = installingPkg === item.package; + + return ( +
+
+
+ + + {item.name} + + + {item.package} + +
+

+ {item.description} +

+
+ + +
+ ); + })} +
+ )} + + {/* Bottom Action Bar */} +
+ + + +
+
+ )} + + {/* ================= PHASE 4B: CONFIGURE PROVIDER ================= */} + {pluginPhase === "configure" && currentConfiguringPlugin && ( +
+
+
+ + Provider {configureIndex + 1} of {newlyInstalledPlugins.length} + + + {currentConfiguringPlugin.manifest?.displayName || currentConfiguringPlugin.name} + +
+

+ Configure Provider Credentials +

+

+ Set up connection credentials for this provider, or skip to configure it later in the dashboard. +

+
+ +
+ {providerError && ( +
+ + {providerError} +
+ )} + + {/* Provider ID */} +
+ + setProviderId(e.target.value)} + placeholder="my-provider-id" + required + disabled={isSavingProvider} + className="h-10 text-sm font-mono" + /> +

+ Unique identifier used when sending notifications through this provider. +

+
+ + {/* Required Credentials from Manifest */} + {currentConfiguringPlugin.manifest?.requiredCredentials && + currentConfiguringPlugin.manifest.requiredCredentials.length > 0 ? ( +
+ + Required Credentials + + {currentConfiguringPlugin.manifest.requiredCredentials.map((credKey) => ( +
+ + + setCredentials((prev) => ({ + ...prev, + [credKey]: e.target.value, + })) + } + disabled={isSavingProvider} + className="h-10 text-sm" + /> +
+ ))} +
+ ) : ( + /* Default generic API Key / Secret for mock or simple providers */ +
+ + + setCredentials((prev) => ({ + ...prev, + apiKey: e.target.value, + })) + } + disabled={isSavingProvider} + className="h-10 text-sm" + /> +
+ )} + + {/* Provider Buttons */} +
+ + + +
+
+
+ )} + + {/* ================= PHASE 4C: COMPLETE ================= */} + {pluginPhase === "complete" && ( +
+
+ +
+ +
+

+ You're All Set! +

+

+ Your master account and initial providers are configured. You are ready to start sending notifications with SimpleNS. +

+
+ +
+ +
+
+ )} + + )} + +
+ + {/* Bottom Step Dots on Mobile */} +
+ {steps.map((s) => ( +
s.number + ? "w-2 bg-primary/50" + : "w-2 bg-muted" + }`} + /> + ))} +
); From 149a4e16ad7757e3d54bc8425bae92b315a59cbb Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Fri, 18 Sep 2026 19:23:59 +0530 Subject: [PATCH 17/46] feat: enhance plugin installation handling and sanitize provider IDs --- dashboard/app/setup/page.tsx | 46 ++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/dashboard/app/setup/page.tsx b/dashboard/app/setup/page.tsx index b64b72e..09f9fd6 100644 --- a/dashboard/app/setup/page.tsx +++ b/dashboard/app/setup/page.tsx @@ -81,7 +81,18 @@ export default function SetupPage() { const [isCatalogLoading, setIsCatalogLoading] = useState(false); const [installedPackages, setInstalledPackages] = useState>(new Set()); const [newlyInstalledPlugins, setNewlyInstalledPlugins] = useState([]); - const [installingPkg, setInstallingPkg] = useState(null); + const [installingPkgs, setInstallingPkgs] = useState>(new Set()); + + // Helper to sanitize provider IDs without slashes ('/') + const sanitizeProviderId = (rawName: string): string => { + return rawName + .replace(/^@/, "") + .replace(/[/\\_ ]/g, "-") + .replace(/^plugin-/, "") + .replace(/-+/g, "-") + .toLowerCase() + + "-provider"; + }; // Provider configuration state const [configureIndex, setConfigureIndex] = useState(0); @@ -217,19 +228,26 @@ export default function SetupPage() { }; const handleInstallPlugin = async (item: PluginCatalogItem) => { - setInstallingPkg(item.package); + setInstallingPkgs((prev) => new Set(prev).add(item.package)); try { const res = await pluginService.install(item.package); - setInstalledPackages((prev) => new Set([...prev, item.package])); + setInstalledPackages((prev) => new Set(prev).add(item.package)); if (res.plugin) { - setNewlyInstalledPlugins((prev) => [...prev, res.plugin]); + setNewlyInstalledPlugins((prev) => { + if (prev.some((p) => p.name === res.plugin.name)) return prev; + return [...prev, res.plugin]; + }); } toast.success(`Installed ${item.name} successfully!`); } catch (err: unknown) { const msg = err instanceof Error ? err.message : "Failed to install plugin"; toast.error(msg); } finally { - setInstallingPkg(null); + setInstallingPkgs((prev) => { + const next = new Set(prev); + next.delete(item.package); + return next; + }); } }; @@ -240,7 +258,7 @@ export default function SetupPage() { } setConfigureIndex(0); const first = newlyInstalledPlugins[0]; - setProviderId(`${first.name.replace(/^plugin-/, "")}-provider`); + setProviderId(sanitizeProviderId(first.name)); setCredentials({}); setProviderError(""); setPluginPhase("configure"); @@ -256,11 +274,15 @@ export default function SetupPage() { return; } - const trimmedId = providerId.trim(); + const trimmedId = providerId.trim().replace(/[/\\ ]/g, "-"); if (!trimmedId) { setProviderError("Provider identifier is required."); return; } + if (trimmedId.includes("/")) { + setProviderError("Provider identifier cannot contain slashes ('/')."); + return; + } setIsSavingProvider(true); @@ -279,7 +301,7 @@ export default function SetupPage() { const nextIndex = configureIndex + 1; const nextPlugin = newlyInstalledPlugins[nextIndex]; setConfigureIndex(nextIndex); - setProviderId(`${nextPlugin.name.replace(/^plugin-/, "")}-provider`); + setProviderId(sanitizeProviderId(nextPlugin.name)); setCredentials({}); setProviderError(""); } else { @@ -298,7 +320,7 @@ export default function SetupPage() { const nextIndex = configureIndex + 1; const nextPlugin = newlyInstalledPlugins[nextIndex]; setConfigureIndex(nextIndex); - setProviderId(`${nextPlugin.name.replace(/^plugin-/, "")}-provider`); + setProviderId(sanitizeProviderId(nextPlugin.name)); setCredentials({}); setProviderError(""); } else { @@ -937,7 +959,7 @@ export default function SetupPage() {
{catalog.map((item) => { const isInstalled = installedPackages.has(item.package); - const isInstalling = installingPkg === item.package; + const isInstalling = installingPkgs.has(item.package); return (
setProviderId(e.target.value)} + onChange={(e) => setProviderId(e.target.value.replace(/[/\\ ]/g, "-"))} placeholder="my-provider-id" required disabled={isSavingProvider} className="h-10 text-sm font-mono" />

- Unique identifier used when sending notifications through this provider. + Unique identifier without slashes (e.g. gmail-provider).

From 33a49d69f258b9a34645cb1a1ce1bce049cad000 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sat, 19 Sep 2026 18:11:57 +0530 Subject: [PATCH 18/46] feat: remove hardcoded config file mounts and enhance environment variable handling --- docker-compose/docker-compose.dev.yaml | 7 ++----- docker-compose/docker-compose.infra.yaml | 2 +- docker-compose/docker-compose.local.yaml | 12 +++--------- docker-compose/docker-compose.master.yaml | 7 ++----- 4 files changed, 8 insertions(+), 20 deletions(-) diff --git a/docker-compose/docker-compose.dev.yaml b/docker-compose/docker-compose.dev.yaml index 4bf71f5..60d403a 100644 --- a/docker-compose/docker-compose.dev.yaml +++ b/docker-compose/docker-compose.dev.yaml @@ -27,7 +27,6 @@ services: SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} volumes: - logs-data-dev:/app/logs - - ../simplens.config.yaml:/app/simplens.config.yaml:ro command: [ "node", "dist/api/server.js" ] networks: - simplens-infra @@ -87,7 +86,6 @@ services: LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-dev:/app/logs - - ../simplens.config.yaml:/app/simplens.config.yaml:ro command: [ "node", "dist/processors/unified/unified.processor.js" ] depends_on: api: @@ -166,10 +164,9 @@ services: PORT: ${DASHBOARD_PORT:-3002} API_BASE_URL: http://api:${PORT:-3000} MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 - AUTH_SECRET: ${AUTH_SECRET:-change-this-in-production} - ADMIN_USERNAME: ${ADMIN_USERNAME:-admin} - ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin} + AUTH_SECRET: ${AUTH_SECRET:-${NS_API_KEY:-change-this-in-production}} AUTH_TRUST_HOST: "true" + HTTPS_COOKIE: ${HTTPS_COOKIE:-false} WEBHOOK_HOST: dashboard WEBHOOK_PORT: "3002" depends_on: diff --git a/docker-compose/docker-compose.infra.yaml b/docker-compose/docker-compose.infra.yaml index 86553c2..5b5483e 100644 --- a/docker-compose/docker-compose.infra.yaml +++ b/docker-compose/docker-compose.infra.yaml @@ -4,7 +4,7 @@ # Shared across all application environments (local, dev, master). # ============================================================================== -name: infra +name: simplens-infra services: # ============================================ diff --git a/docker-compose/docker-compose.local.yaml b/docker-compose/docker-compose.local.yaml index f8732e6..61b3c48 100644 --- a/docker-compose/docker-compose.local.yaml +++ b/docker-compose/docker-compose.local.yaml @@ -27,10 +27,8 @@ services: BROKERS: kafka:9093 LOKI_URL: http://loki:3100 LOG_LEVEL: ${LOG_LEVEL:-info} - SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} volumes: - logs-data-local:/app/logs - # - ../simplens.config.yaml:/app/simplens.config.yaml:ro command: [ "node", "dist/api/server.js" ] networks: - simplens-infra @@ -84,13 +82,10 @@ services: MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 REDIS_URL: redis://redis:6379 BROKERS: kafka:9093 - PROCESSOR_CHANNEL: ${PROCESSOR_CHANNEL:-all} - SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} LOKI_URL: http://loki:3100 LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-local:/app/logs - # - ../simplens.config.yaml:/app/simplens.config.yaml:ro command: [ "node", "dist/processors/unified/unified.processor.js" ] depends_on: api: @@ -168,14 +163,13 @@ services: env_file: - ../.env environment: - NODE_ENV: production + NODE_ENV: ${NODE_ENV:-development} PORT: ${DASHBOARD_PORT:-3002} API_BASE_URL: http://api:${PORT:-3000} MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 - AUTH_SECRET: ${AUTH_SECRET:-change-this-in-production} - ADMIN_USERNAME: ${ADMIN_USERNAME:-admin} - ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin} AUTH_TRUST_HOST: "true" + AUTH_SECRET: ${AUTH_SECRET:-${NS_API_KEY:-simplens-default-session-secret-change-in-production}} + HTTPS_COOKIE: ${HTTPS_COOKIE:-false} WEBHOOK_HOST: dashboard WEBHOOK_PORT: "3002" depends_on: diff --git a/docker-compose/docker-compose.master.yaml b/docker-compose/docker-compose.master.yaml index 7643cc4..dc4831c 100644 --- a/docker-compose/docker-compose.master.yaml +++ b/docker-compose/docker-compose.master.yaml @@ -27,7 +27,6 @@ services: SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} volumes: - logs-data-master:/app/logs - - ../simplens.config.yaml:/app/simplens.config.yaml:ro command: [ "node", "dist/api/server.js" ] networks: - simplens-infra @@ -87,7 +86,6 @@ services: LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-master:/app/logs - - ../simplens.config.yaml:/app/simplens.config.yaml:ro command: [ "node", "dist/processors/unified/unified.processor.js" ] depends_on: api: @@ -166,10 +164,9 @@ services: PORT: ${DASHBOARD_PORT:-3002} API_BASE_URL: http://api:${PORT:-3000} MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 - AUTH_SECRET: ${AUTH_SECRET:-change-this-in-production} - ADMIN_USERNAME: ${ADMIN_USERNAME:-admin} - ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin} + AUTH_SECRET: ${AUTH_SECRET:-${NS_API_KEY:-change-this-in-production}} AUTH_TRUST_HOST: "true" + HTTPS_COOKIE: ${HTTPS_COOKIE:-false} WEBHOOK_HOST: dashboard WEBHOOK_PORT: "3002" depends_on: From d324e00b4030ecb3f56b32e7f06f154b5619e9c2 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sat, 19 Sep 2026 23:17:32 +0530 Subject: [PATCH 19/46] feat: Implement npm authentication service and plugin installation enhancements - Added NpmAuthService to manage npm authentication tokens and registry configurations. - Introduced new API endpoints for retrieving, saving, and deleting npm auth configurations. - Enhanced plugin installation process to support private packages with authentication. - Implemented validation for SimpleNSProvider to ensure compliance with required methods and manifest structure. - Updated schemas for npm auth configuration and plugin installation payloads. - Added integration and unit tests for the new npm auth service and plugin validation logic. --- .../plugins/installed-plugins-tab.tsx | 374 ++++++++++++++++-- dashboard/lib/api-client.ts | 12 +- dashboard/lib/types.ts | 16 + src/api/controllers/plugins.controller.ts | 98 ++++- src/api/routes/plugins.routes.ts | 6 + src/api/server.ts | 3 + src/plugins/index.ts | 2 + src/plugins/loader/plugin-fs.ts | 80 +++- src/plugins/services/npm-auth.service.ts | 203 ++++++++++ .../services/plugin-manager.service.ts | 36 +- src/processors/unified/unified.processor.ts | 3 + src/types/schemas.ts | 30 ++ src/types/types.ts | 9 +- tests/integration/api/plugins.test.ts | 249 ++++++++++++ .../workers/consumers/status.consumer.test.ts | 2 +- tests/unit/plugins/npm-auth.service.test.ts | 139 +++++++ .../plugins/plugin-manager.service.test.ts | 14 + tests/unit/plugins/plugin-validation.test.ts | 85 ++++ tsconfig.json | 3 +- 19 files changed, 1309 insertions(+), 55 deletions(-) create mode 100644 src/plugins/services/npm-auth.service.ts create mode 100644 tests/integration/api/plugins.test.ts create mode 100644 tests/unit/plugins/npm-auth.service.test.ts create mode 100644 tests/unit/plugins/plugin-validation.test.ts diff --git a/dashboard/components/plugins/installed-plugins-tab.tsx b/dashboard/components/plugins/installed-plugins-tab.tsx index 1ebb0e3..3dbbafe 100644 --- a/dashboard/components/plugins/installed-plugins-tab.tsx +++ b/dashboard/components/plugins/installed-plugins-tab.tsx @@ -3,12 +3,13 @@ import { useMemo, useState } from "react"; import useSWR from "swr"; import { pluginService } from "@/lib/api-client"; -import type { InstalledPlugin, PluginCatalogItem } from "@/lib/types"; +import type { InstalledPlugin, PluginCatalogItem, InstallPluginPayload, NpmAuthStatus } from "@/lib/types"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; import { Badge } from "@/components/ui/badge"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; import { Dialog, DialogContent, @@ -30,7 +31,7 @@ import { import { Skeleton } from "@/components/ui/skeleton"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { Download, Package, RefreshCw, Trash2, ArrowUpDown, Loader2, ExternalLink, Info, Search } from "lucide-react"; +import { Download, Package, RefreshCw, Trash2, ArrowUpDown, Loader2, ExternalLink, Info, Search, Lock, ShieldCheck, Eye, EyeOff, Check } from "lucide-react"; import { toast } from "sonner"; export function InstalledPluginsTab() { @@ -40,10 +41,26 @@ export function InstalledPluginsTab() { ); const [installModalOpen, setInstallModalOpen] = useState(false); - const [catalogTab, setCatalogTab] = useState<"official" | "community">("official"); + const [catalogTab, setCatalogTab] = useState<"official" | "community" | "custom">("official"); const [catalogSearch, setCatalogSearch] = useState(""); const [selectedVersions, setSelectedVersions] = useState>({}); - const [installingPackage, setInstallingPackage] = useState(null); + const [installingPkgs, setInstallingPkgs] = useState>(new Set()); + + // Custom Package Form State + const [customPkg, setCustomPkg] = useState(""); + const [customVersion, setCustomVersion] = useState(""); + const [isPrivate, setIsPrivate] = useState(false); + const [useSavedToken, setUseSavedToken] = useState(true); + const [npmToken, setNpmToken] = useState(""); + const [showToken, setShowToken] = useState(false); + const [enterpriseRegistryUrl, setEnterpriseRegistryUrl] = useState(""); + const [saveToken, setSaveToken] = useState(true); + const [isInstallingCustom, setIsInstallingCustom] = useState(false); + + const { data: npmAuth, mutate: mutateNpmAuth } = useSWR( + installModalOpen ? "/api/plugins/npm-auth" : null, + () => pluginService.getNpmAuth() + ); const { data: officialCatalog = [], isLoading: officialLoading, error: officialError } = useSWR( installModalOpen && catalogTab === "official" ? "plugin-catalog-official" : null, @@ -66,7 +83,7 @@ export function InstalledPluginsTab() { const handleInstall = async (plugin: PluginCatalogItem) => { const version = selectedVersions[plugin.package] || "latest"; - setInstallingPackage(plugin.package); + setInstallingPkgs((prev) => new Set(prev).add(plugin.package)); try { await pluginService.install(plugin.package, version === "latest" ? undefined : version); toast.success(`Plugin '${plugin.name}' installed successfully`); @@ -75,13 +92,70 @@ export function InstalledPluginsTab() { const message = err instanceof Error ? err.message : "Failed to install plugin"; toast.error(message); } finally { - setInstallingPackage(null); + setInstallingPkgs((prev) => { + const next = new Set(prev); + next.delete(plugin.package); + return next; + }); + } + }; + + const handleInstallCustom = async (e: React.FormEvent) => { + e.preventDefault(); + const pkgName = customPkg.trim(); + if (!pkgName) { + toast.error("Please enter a package name"); + return; + } + + const payload: InstallPluginPayload = { + package: pkgName, + version: customVersion.trim() || undefined, + }; + + if (isPrivate) { + const token = useSavedToken && npmAuth?.is_configured ? undefined : npmToken.trim(); + if (!useSavedToken && !token) { + toast.error("Please enter an npm authentication token for this private package"); + return; + } + if (token || enterpriseRegistryUrl.trim()) { + payload.auth = { + token: token || undefined, + registry_url: enterpriseRegistryUrl.trim() || undefined, + save_token: saveToken, + }; + } + } + + setIsInstallingCustom(true); + setInstallingPkgs((prev) => new Set(prev).add(pkgName)); + try { + await pluginService.install(payload); + toast.success(`Plugin '${pkgName}' installed successfully!`); + mutate(); + if (payload.auth?.token && saveToken) { + mutateNpmAuth(); + } + setCustomPkg(""); + setCustomVersion(""); + setNpmToken(""); + setIsPrivate(false); + setInstallModalOpen(false); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to install custom plugin"; + toast.error(message); + } finally { + setIsInstallingCustom(false); + setInstallingPkgs((prev) => { + const next = new Set(prev); + next.delete(pkgName); + return next; + }); } }; const catalog = catalogTab === "official" ? officialCatalog : communityCatalog; - const catalogLoading = catalogTab === "official" ? officialLoading : communityLoading; - const catalogError = catalogTab === "official" ? officialError : communityError; const filteredCatalog = useMemo(() => { const query = catalogSearch.trim().toLowerCase(); if (!query) return catalog; @@ -336,37 +410,111 @@ export function InstalledPluginsTab() { Install Plugin Package - Browse official and community provider packages available for SimpleNS. + Browse official and community provider packages or install custom plugins from npmjs.
-
- - setCatalogSearch(event.target.value)} - /> -
- setCatalogTab(value as "official" | "community")}> - + setCatalogTab(value as "official" | "community" | "custom")}> + Official Community + Custom Package - - {catalogLoading ? ( + + {catalogTab !== "custom" && ( +
+ + setCatalogSearch(event.target.value)} + /> +
+ )} + + + {officialLoading ? (
Loading plugins...
- ) : catalogError ? ( + ) : officialError ? (
- Failed to load {catalogTab} plugins. + Failed to load official plugins.
) : filteredCatalog.length === 0 ? (
- {catalogTab === "community" && catalog.length === 0 + No plugins match your search. +
+ ) : ( +
+ {filteredCatalog.map((plugin) => { + const versions = plugin.versions?.length ? plugin.versions : ["latest"]; + const selectedVersion = selectedVersions[plugin.package] || versions[0]; + const isInstalled = installedPackages.has(plugin.package); + const isInstalling = installingPkgs.has(plugin.package); + return ( +
+
+
{plugin.name}
+
{plugin.package}
+

{plugin.description}

+
+
+ + {isInstalled ? ( + + ) : ( + + )} +
+
+ ); + })} +
+ )} + + + + {communityLoading ? ( +
+ Loading plugins... +
+ ) : communityError ? ( +
+ Failed to load community plugins. +
+ ) : filteredCatalog.length === 0 ? ( +
+ {communityCatalog.length === 0 ? "No community plugins are available yet." : "No plugins match your search."}
@@ -375,6 +523,8 @@ export function InstalledPluginsTab() { {filteredCatalog.map((plugin) => { const versions = plugin.versions?.length ? plugin.versions : ["latest"]; const selectedVersion = selectedVersions[plugin.package] || versions[0]; + const isInstalled = installedPackages.has(plugin.package); + const isInstalling = installingPkgs.has(plugin.package); return (
@@ -396,8 +546,9 @@ export function InstalledPluginsTab() { ))} - {installedPackages.has(plugin.package) ? ( - ) : ( @@ -405,9 +556,9 @@ export function InstalledPluginsTab() { type="button" size="sm" onClick={() => void handleInstall(plugin)} - disabled={installingPackage !== null} + disabled={isInstalling} > - {installingPackage === plugin.package ? ( + {isInstalling ? ( ) : ( @@ -422,10 +573,171 @@ export function InstalledPluginsTab() {
)} + + +
+
+
+ + Install from npmjs or Enterprise Registry +
+

+ Install any custom plugin implementing the SimpleNS Provider interface. For scoped packages (e.g. @myenterprise/provider), the scope is automatically detected. +

+
+ +
+
+ + setCustomPkg(e.target.value)} + required + disabled={isInstallingCustom} + /> +
+ +
+ + setCustomVersion(e.target.value)} + disabled={isInstallingCustom} + /> +
+
+ + {/* Private / Enterprise npm Package Toggle */} +
+
+
+
+ + Private / Enterprise Package +
+

+ Enable if this package requires an npm authentication token to download. +

+
+ +
+ + {isPrivate && ( +
+ {npmAuth?.is_configured && ( +
+
+ + + Saved token active: {npmAuth.masked_token} + +
+ +
+ )} + + {(!useSavedToken || !npmAuth?.is_configured) && ( +
+
+ +
+ setNpmToken(e.target.value)} + required={isPrivate && (!useSavedToken || !npmAuth?.is_configured)} + className="pr-10 font-mono text-xs" + disabled={isInstallingCustom} + /> + +
+
+ +
+ + setEnterpriseRegistryUrl(e.target.value)} + disabled={isInstallingCustom} + /> +
+ +
+ setSaveToken(e.target.checked)} + className="rounded text-primary h-4 w-4" + disabled={isInstallingCustom} + /> + +
+
+ )} +
+ )} +
+ +
+ +
+
+
- diff --git a/dashboard/lib/api-client.ts b/dashboard/lib/api-client.ts index d085872..b3f5a1c 100644 --- a/dashboard/lib/api-client.ts +++ b/dashboard/lib/api-client.ts @@ -21,6 +21,8 @@ import { AdminChannelProviderMeta, InstalledPlugin, PluginCatalogItem, + NpmAuthStatus, + InstallPluginPayload, ProviderDto, ChannelRoutingDto, OperationalSettings, @@ -150,12 +152,18 @@ export const pluginService = { apiClient.get(`/api/plugins/catalog/${category}`), getMetadata: (): Promise => apiClient.get('/api/plugins'), listInstalled: (): Promise<{ plugins: InstalledPlugin[] }> => apiClient.get('/api/plugins/installed'), - install: (packageName: string, version?: string): Promise<{ message: string; plugin: InstalledPlugin }> => - apiClient.post('/api/plugins/install', { package: packageName, version }), + install: (payload: string | InstallPluginPayload, version?: string): Promise<{ message: string; plugin: InstalledPlugin }> => { + const body = typeof payload === 'string' ? { package: payload, version } : payload; + return apiClient.post('/api/plugins/install', body); + }, changeVersion: (packageName: string, version: string): Promise<{ message: string; plugin: InstalledPlugin }> => apiClient.put('/api/plugins/version', { package: packageName, version }), uninstall: (packageName: string): Promise<{ message: string }> => apiClient.delete(`/api/plugins/${encodeURIComponent(packageName)}`), + getNpmAuth: (): Promise => apiClient.get('/api/plugins/npm-auth'), + saveNpmAuth: (payload: { token: string; registry_url?: string }): Promise<{ message: string; status: NpmAuthStatus }> => + apiClient.post('/api/plugins/npm-auth', payload), + deleteNpmAuth: (): Promise<{ message: string }> => apiClient.delete('/api/plugins/npm-auth'), }; export const providerService = { diff --git a/dashboard/lib/types.ts b/dashboard/lib/types.ts index 1f80942..b483299 100644 --- a/dashboard/lib/types.ts +++ b/dashboard/lib/types.ts @@ -251,6 +251,22 @@ export interface PluginCatalogItem { versions?: string[]; } +export interface NpmAuthStatus { + is_configured: boolean; + registry_url?: string; + masked_token?: string; +} + +export interface InstallPluginPayload { + package: string; + version?: string; + auth?: { + token?: string; + registry_url?: string; + save_token?: boolean; + }; +} + export interface ProviderDto { _id?: string; id: string; diff --git a/src/api/controllers/plugins.controller.ts b/src/api/controllers/plugins.controller.ts index 7d08e6e..fc19559 100644 --- a/src/api/controllers/plugins.controller.ts +++ b/src/api/controllers/plugins.controller.ts @@ -4,10 +4,11 @@ * Exposes plugin metadata, package management, and status */ -import { Request, Response } from 'express'; -import { PluginRegistry, PluginManagerService } from '@src/plugins/index.js'; +import type { Request, Response } from 'express'; +import { PluginRegistry, PluginManagerService, NpmAuthService } from '@src/plugins/index.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; import axios from 'axios'; +import { installPluginPayloadSchema, npmAuthConfigSchema } from '@src/types/schemas.js'; import type { plugin_catalog_entry } from '@src/types/types.js'; const PLUGIN_CATALOG_BASE_URL = 'https://www.simplens.in/plugins'; @@ -91,16 +92,18 @@ export const listPluginCatalog = async (req: Request, res: Response): Promise => { try { - const { package: packageName, version } = req.body; - if (!packageName || typeof packageName !== 'string') { + const validation = installPluginPayloadSchema.safeParse(req.body); + if (!validation.success) { res.status(400).json({ error: 'Bad Request', - message: 'A valid npm package name is required' + message: 'Invalid plugin installation payload', + details: validation.error.flatten().fieldErrors, }); return; } - const plugin = await PluginManagerService.installPlugin(packageName.trim(), version); + const { package: packageName, version, auth } = validation.data; + const plugin = await PluginManagerService.installPlugin(packageName.trim(), version?.trim(), auth); res.status(201).json({ message: `Plugin '${packageName}' installed successfully`, plugin @@ -108,8 +111,16 @@ export const installPlugin = async (req: Request, res: Response): Promise } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); logger.error(`Error installing plugin: ${message}`); - res.status(500).json({ - error: 'Failed to install plugin', + + const isBadRequest = + message.includes('not a valid SimpleNS plugin') || + message.includes('invalid plugin manifest') || + message.includes('npm authentication failed') || + message.includes('not found') || + message.includes('Missing required provider methods'); + + res.status(isBadRequest ? 400 : 500).json({ + error: isBadRequest ? 'Bad Request' : 'Failed to install plugin', message }); } @@ -183,3 +194,74 @@ export const uninstallPlugin = async (req: Request, res: Response): Promise => { + try { + const status = await NpmAuthService.getAuthStatus(); + res.json(status); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error retrieving npm auth status: ${message}`); + res.status(500).json({ + error: 'Failed to retrieve npm auth status', + message, + }); + } +}; + +/** + * POST /api/plugins/npm-auth + * Configure/save npm auth token and registry URL + */ +export const saveNpmAuth = async (req: Request, res: Response): Promise => { + try { + const validation = npmAuthConfigSchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + error: 'Bad Request', + message: 'Invalid npm auth configuration', + details: validation.error.flatten().fieldErrors, + }); + return; + } + + const { token, registry_url } = validation.data; + const status = await NpmAuthService.saveNpmAuth(token, registry_url); + res.status(200).json({ + message: 'npm auth configuration saved successfully', + status, + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error saving npm auth configuration: ${message}`); + res.status(500).json({ + error: 'Failed to save npm auth configuration', + message, + }); + } +}; + +/** + * DELETE /api/plugins/npm-auth + * Delete saved npm auth configuration + */ +export const deleteNpmAuth = async (_req: Request, res: Response): Promise => { + try { + await NpmAuthService.deleteNpmAuth(); + res.json({ + message: 'npm auth configuration removed successfully', + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error deleting npm auth configuration: ${message}`); + res.status(500).json({ + error: 'Failed to delete npm auth configuration', + message, + }); + } +}; + diff --git a/src/api/routes/plugins.routes.ts b/src/api/routes/plugins.routes.ts index 67d379c..a16d296 100644 --- a/src/api/routes/plugins.routes.ts +++ b/src/api/routes/plugins.routes.ts @@ -12,6 +12,9 @@ import { installPlugin, changePluginVersion, uninstallPlugin, + getNpmAuth, + saveNpmAuth, + deleteNpmAuth, } from '../controllers/plugins.controller.js'; const router = Router(); @@ -19,6 +22,9 @@ const router = Router(); router.get('/', getPluginsMetadata); router.get('/installed', listInstalledPlugins); router.get('/catalog/:category', listPluginCatalog); +router.get('/npm-auth', getNpmAuth); +router.post('/npm-auth', saveNpmAuth); +router.delete('/npm-auth', deleteNpmAuth); router.post('/install', installPlugin); router.put('/version', changePluginVersion); router.delete('/:scope/:package', uninstallPlugin); diff --git a/src/api/server.ts b/src/api/server.ts index 6ea5244..267f707 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -24,6 +24,7 @@ import { apiLogger as logger } from '@src/workers/utils/logger.js'; import { PluginSyncService, PluginManagerService, + NpmAuthService, ProviderManagerService, ChannelRoutingService, YamlMigrator, @@ -83,6 +84,8 @@ const start_server = async () => { // 1. Initialize Redis subscriber & sync handlers for plugins await PluginSyncService.startSubscriber(); + NpmAuthService.registerSyncHandlers(); + await NpmAuthService.syncNpmrc(); PluginManagerService.registerSyncHandlers(); ProviderManagerService.registerSyncHandlers(); ChannelRoutingService.registerSyncHandlers(); diff --git a/src/plugins/index.ts b/src/plugins/index.ts index b076987..264a63e 100644 --- a/src/plugins/index.ts +++ b/src/plugins/index.ts @@ -36,6 +36,7 @@ export { // Dynamic Management Services export { PluginManagerService } from './services/plugin-manager.service.js'; +export { NpmAuthService } from './services/npm-auth.service.js'; export { ProviderManagerService, type ProviderResponseDto } from './services/provider-manager.service.js'; export { ChannelRoutingService } from './services/channel-routing.service.js'; export { YamlMigrator } from './bootstrap/yaml-migrator.js'; @@ -46,4 +47,5 @@ export { uninstallNpmPackage, extractPackageManifest, importAndInstantiateProvider, + validateSimpleNSProvider, } from './loader/plugin-fs.js'; diff --git a/src/plugins/loader/plugin-fs.ts b/src/plugins/loader/plugin-fs.ts index f2029f6..c84d139 100644 --- a/src/plugins/loader/plugin-fs.ts +++ b/src/plugins/loader/plugin-fs.ts @@ -10,6 +10,7 @@ import { join } from 'path'; import { execSync } from 'child_process'; import { pathToFileURL } from 'url'; import type { SimpleNSProvider, ProviderManifest } from '@src/types/types.js'; +import { providerManifestSchema } from '@src/types/schemas.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; export const PLUGINS_DIR = join(process.cwd(), '.plugins'); @@ -189,6 +190,59 @@ export async function importAndInstantiateProvider(packageName: string): Promise } } +/** + * Validate that an instantiated object strictly satisfies the SimpleNSProvider contract + */ +export function validateSimpleNSProvider( + provider: unknown, + packageName: string +): SimpleNSProvider { + if (!provider || typeof provider !== 'object') { + throw new Error( + `Package '${packageName}' is not a valid SimpleNS plugin: default export must be a Provider object or class.` + ); + } + + const p = provider as Record; + + if (!p.manifest || typeof p.manifest !== 'object') { + throw new Error( + `Package '${packageName}' is not a valid SimpleNS plugin: missing 'manifest' definition.` + ); + } + + const manifestValidation = providerManifestSchema.safeParse(p.manifest); + if (!manifestValidation.success) { + const errorDetails = Object.entries(manifestValidation.error.flatten().fieldErrors) + .map(([k, v]) => `${k}: ${v?.join(', ')}`) + .join('; '); + throw new Error( + `Package '${packageName}' has an invalid plugin manifest: ${errorDetails}` + ); + } + + const requiredMethods: (keyof SimpleNSProvider)[] = [ + 'send', + 'initialize', + 'healthCheck', + 'shutdown', + 'getNotificationSchema', + 'getRecipientSchema', + 'getContentSchema', + 'getRateLimitConfig', + ]; + + const missingMethods = requiredMethods.filter((method) => typeof p[method] !== 'function'); + + if (missingMethods.length > 0) { + throw new Error( + `Package '${packageName}' is not a valid SimpleNS plugin. Missing required provider methods: ${missingMethods.join(', ')}` + ); + } + + return provider as SimpleNSProvider; +} + /** * Extract manifest and authoritative package.json version from an installed package */ @@ -209,10 +263,8 @@ export async function extractPackageManifest( const pkgJson = JSON.parse(readFileSync(pkgJsonPath, 'utf-8')); const version: string = pkgJson.version || '1.0.0'; - const provider = await importAndInstantiateProvider(packageName); - if (!provider.manifest) { - throw new Error(`Plugin '${packageName}' does not export a valid manifest property.`); - } + const rawProvider = await importAndInstantiateProvider(packageName); + const provider = validateSimpleNSProvider(rawProvider, packageName); // Ensure manifest version matches package.json const manifest: ProviderManifest = { @@ -237,9 +289,23 @@ export function installNpmPackage(packageName: string, version?: string): void { stdio: 'pipe', }); logger.success(`Successfully installed ${specifier}`); - } catch (err) { - logger.error(`Failed to install ${specifier}:`, err); - throw new Error(`Failed to install package: ${specifier}`, { cause: err }); + } catch (err: unknown) { + const execErr = err as { stderr?: Buffer; message?: string }; + const stderrMsg = execErr.stderr ? execErr.stderr.toString('utf-8') : ''; + logger.error(`Failed to install ${specifier}:`, stderrMsg || execErr.message || err); + + let friendlyMessage = `Failed to install package '${specifier}'`; + if (stderrMsg.includes('E401') || stderrMsg.includes('401 Unauthorized')) { + friendlyMessage = `npm authentication failed (401 Unauthorized) for package '${packageName}'. Verify your npm token.`; + } else if (stderrMsg.includes('E404') || stderrMsg.includes('404 Not Found')) { + friendlyMessage = `Package '${packageName}' was not found (404 Not Found) on npm registry.`; + } else if (stderrMsg.includes('ETARGET')) { + friendlyMessage = `Version '${version}' not found for package '${packageName}'.`; + } else if (stderrMsg.trim()) { + friendlyMessage += `: ${stderrMsg.trim().slice(0, 300)}`; + } + + throw new Error(friendlyMessage, { cause: err }); } } diff --git a/src/plugins/services/npm-auth.service.ts b/src/plugins/services/npm-auth.service.ts new file mode 100644 index 0000000..28e58d8 --- /dev/null +++ b/src/plugins/services/npm-auth.service.ts @@ -0,0 +1,203 @@ +/** + * NPM Authentication & Registry Configuration Service + * + * Manages npm access tokens and enterprise registry configurations, + * encrypted storage in MongoDB, and synchronization of .plugins/.npmrc. + */ + +import { writeFileSync } from 'fs'; +import { join } from 'path'; +import system_config_model from '@src/database/models/system-config.models.js'; +import { getOrCreateEncryptionKey } from '@src/admin-alerts/key-manager.js'; +import { encrypt, decrypt } from '@src/utils/encryption.utils.js'; +import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; +import { PLUGINS_DIR, initPluginsDir } from '@src/plugins/loader/plugin-fs.js'; +import type { encrypted_config, npm_auth_status } from '@src/types/types.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +export const NPM_AUTH_CONFIG_KEY = 'npm_auth_config'; +export const NPMRC_PATH = join(PLUGINS_DIR, '.npmrc'); + +interface StoredNpmAuthConfig { + encrypted_token: encrypted_config; + registry_url: string; +} + +/** + * Extract npm scope from a package name if present + * E.g. "@simplens/resend" -> "@simplens", "lodash" -> undefined + */ +export function extractScope(packageName: string): string | undefined { + const match = packageName.trim().match(/^(@[a-zA-Z0-9~_.-]+)\//); + return match ? match[1] : undefined; +} + +/** + * Format a registry URL into an .npmrc auth token line + * E.g. "https://registry.npmjs.org/" -> "//registry.npmjs.org/:_authToken=..." + */ +export function toNpmrcTokenLine(registryUrl: string, token: string): string { + try { + const url = new URL(registryUrl); + let hostPath = url.host + url.pathname; + if (!hostPath.endsWith('/')) { + hostPath += '/'; + } + return `//${hostPath}:_authToken=${token.trim()}`; + } catch { + return `//registry.npmjs.org/:_authToken=${token.trim()}`; + } +} + +export class NpmAuthService { + /** + * Retrieve active npm auth credentials from MongoDB or process.env + */ + public static async getDecryptedToken(): Promise<{ token: string; registry_url: string } | null> { + try { + const doc = await system_config_model.findOne({ key: NPM_AUTH_CONFIG_KEY }).lean(); + if (doc?.value) { + const stored = doc.value as StoredNpmAuthConfig; + if (stored.encrypted_token) { + const key = await getOrCreateEncryptionKey(); + const token = decrypt(stored.encrypted_token, key); + return { + token, + registry_url: stored.registry_url || 'https://registry.npmjs.org/', + }; + } + } + } catch (err) { + logger.error('Failed to decrypt stored npm token from MongoDB:', err); + } + + // Fall back to environment variable if present + const envToken = process.env.NPM_TOKEN || process.env.NPM_AUTH_TOKEN; + if (envToken) { + return { + token: envToken, + registry_url: process.env.NPM_REGISTRY_URL || 'https://registry.npmjs.org/', + }; + } + + return null; + } + + /** + * Get public status of configured npm auth (with masked token) + */ + public static async getAuthStatus(): Promise { + const auth = await this.getDecryptedToken(); + if (!auth || !auth.token) { + return { + is_configured: false, + }; + } + + const t = auth.token; + const masked = t.length > 8 + ? `${t.slice(0, 4)}****${t.slice(-4)}` + : '****'; + + return { + is_configured: true, + registry_url: auth.registry_url, + masked_token: masked, + }; + } + + /** + * Save and encrypt npm auth configuration in MongoDB, sync .npmrc, and broadcast + */ + public static async saveNpmAuth(token: string, registryUrl?: string): Promise { + const cleanToken = token.trim(); + const cleanRegistry = registryUrl?.trim() || 'https://registry.npmjs.org/'; + + const key = await getOrCreateEncryptionKey(); + const encryptedToken = encrypt(cleanToken, key); + + const storedValue: StoredNpmAuthConfig = { + encrypted_token: encryptedToken, + registry_url: cleanRegistry, + }; + + await system_config_model.findOneAndUpdate( + { key: NPM_AUTH_CONFIG_KEY }, + { key: NPM_AUTH_CONFIG_KEY, value: storedValue }, + { upsert: true, new: true } + ); + + await this.syncNpmrc(); + await PluginSyncService.publish('NPM_AUTH_UPDATED', {}); + + logger.success(`npm auth configuration saved and broadcasted successfully for ${cleanRegistry}`); + return this.getAuthStatus(); + } + + /** + * Delete saved npm auth configuration, sync .npmrc, and broadcast + */ + public static async deleteNpmAuth(): Promise { + await system_config_model.deleteOne({ key: NPM_AUTH_CONFIG_KEY }); + await this.syncNpmrc(); + await PluginSyncService.publish('NPM_AUTH_UPDATED', {}); + logger.info('npm auth configuration removed and broadcasted'); + } + + /** + * Generate and write .plugins/.npmrc + * Supports optional inline scope/registry/token for one-off private installs + */ + public static async syncNpmrc(temporaryOverride?: { + scope?: string; + registryUrl?: string; + token?: string; + }): Promise { + initPluginsDir(); + + const lines: string[] = [ + '# Auto-generated by SimpleNS - do not edit manually', + 'package-lock=false', + ]; + + const activeAuth = await this.getDecryptedToken(); + + // 1. Configured persistent auth + if (activeAuth && activeAuth.token) { + const isCustomRegistry = activeAuth.registry_url && !activeAuth.registry_url.includes('registry.npmjs.org'); + if (isCustomRegistry) { + lines.push(`registry=${activeAuth.registry_url}`); + } + lines.push(toNpmrcTokenLine(activeAuth.registry_url, activeAuth.token)); + } + + // 2. Temporary/Inline override (e.g. for a specific scoped install) + if (temporaryOverride?.token) { + const targetRegistry = temporaryOverride.registryUrl || 'https://registry.npmjs.org/'; + if (temporaryOverride.scope) { + lines.push(`${temporaryOverride.scope}:registry=${targetRegistry}`); + } else if (!targetRegistry.includes('registry.npmjs.org')) { + lines.push(`registry=${targetRegistry}`); + } + lines.push(toNpmrcTokenLine(targetRegistry, temporaryOverride.token)); + } + + const content = lines.join('\n') + '\n'; + try { + writeFileSync(NPMRC_PATH, content, { encoding: 'utf-8', mode: 0o600 }); + logger.debug(`Synchronized .npmrc in ${PLUGINS_DIR}`); + } catch (err) { + logger.error('Failed to write .npmrc:', err); + } + } + + /** + * Register remote synchronization handlers with Redis Pub/Sub + */ + public static registerSyncHandlers(): void { + PluginSyncService.on('NPM_AUTH_UPDATED', async () => { + logger.info('Sync: Received NPM_AUTH_UPDATED event, refreshing .npmrc...'); + await NpmAuthService.syncNpmrc(); + }); + } +} diff --git a/src/plugins/services/plugin-manager.service.ts b/src/plugins/services/plugin-manager.service.ts index bf9cfc4..fe0ebad 100644 --- a/src/plugins/services/plugin-manager.service.ts +++ b/src/plugins/services/plugin-manager.service.ts @@ -16,7 +16,8 @@ import { } from '@src/plugins/loader/plugin-fs.js'; import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; import { ProviderManagerService } from '@src/plugins/services/provider-manager.service.js'; -import type { plugin_document } from '@src/types/types.js'; +import type { plugin_document, ProviderManifest, install_plugin_payload } from '@src/types/types.js'; +import { NpmAuthService, extractScope } from './npm-auth.service.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; export class PluginManagerService { @@ -32,7 +33,22 @@ export class PluginManagerService { return await PluginSyncService.runWithLock(async () => { try { installNpmPackage(packageName, version); - const { manifest, version: authoritativeVersion } = await extractPackageManifest(packageName); + + let manifest: ProviderManifest; + let authoritativeVersion: string; + try { + const extracted = await extractPackageManifest(packageName); + manifest = extracted.manifest; + authoritativeVersion = extracted.version; + } catch (validationErr) { + logger.warn(`Plugin validation failed for '${packageName}'. Rolling back by uninstalling...`); + try { + uninstallNpmPackage(packageName); + } catch (uninstallErr) { + logger.error(`Rollback failed for '${packageName}':`, uninstallErr); + } + throw validationErr; + } const plugin = await Plugin.findOneAndUpdate( { name: packageName }, @@ -67,8 +83,22 @@ export class PluginManagerService { */ public static async installPlugin( packageName: string, - version?: string + version?: string, + auth?: install_plugin_payload['auth'] ): Promise { + if (auth?.token) { + const scope = extractScope(packageName); + if (auth.save_token !== false) { + await NpmAuthService.saveNpmAuth(auth.token, auth.registry_url); + } else { + await NpmAuthService.syncNpmrc({ + scope, + registryUrl: auth.registry_url, + token: auth.token, + }); + } + } + logger.info(`Installing plugin package '${packageName}' (version: ${version || 'latest'})...`); return this.saveAndBroadcastPlugin(packageName, version, 'PLUGIN_INSTALLED', true); } diff --git a/src/processors/unified/unified.processor.ts b/src/processors/unified/unified.processor.ts index 46549e5..481a144 100644 --- a/src/processors/unified/unified.processor.ts +++ b/src/processors/unified/unified.processor.ts @@ -16,6 +16,7 @@ import { PluginRegistry, PluginSyncService, PluginManagerService, + NpmAuthService, ProviderManagerService, ChannelRoutingService, loadProvidersFromDatabase, @@ -201,6 +202,8 @@ const main = async (): Promise => { // 3. Initialize Redis Pub/Sub synchronization await PluginSyncService.startSubscriber(); + NpmAuthService.registerSyncHandlers(); + await NpmAuthService.syncNpmrc(); PluginManagerService.registerSyncHandlers(); ProviderManagerService.registerSyncHandlers(); ChannelRoutingService.registerSyncHandlers(); diff --git a/src/types/schemas.ts b/src/types/schemas.ts index 71970ce..42f6cbe 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -554,3 +554,33 @@ export const adminLoginSchema = z.object({ password: z.string().min(1, "Password is required"), }); +/** + * npm registry authentication config schema + */ +export const npmAuthConfigSchema = z.object({ + token: z.string().min(1, "npm token cannot be empty"), + registry_url: z.string().url("Invalid registry URL").default("https://registry.npmjs.org/"), +}); + +/** + * npm registry auth status schema (safe to return to UI with masked token) + */ +export const npmAuthStatusSchema = z.object({ + is_configured: z.boolean(), + registry_url: z.string().optional(), + masked_token: z.string().optional(), +}); + +/** + * Payload schema for installing a plugin package + */ +export const installPluginPayloadSchema = z.object({ + package: z.string().min(1, "Package name is required"), + version: z.string().optional(), + auth: z.object({ + token: z.string().optional(), + registry_url: z.string().url("Invalid registry URL").optional(), + save_token: z.boolean().default(true), + }).optional(), +}); + diff --git a/src/types/types.ts b/src/types/types.ts index 0bb1594..fbf1324 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -132,6 +132,9 @@ import { partialOperationalSettingsSchema, adminSetupSchema, adminLoginSchema, + npmAuthConfigSchema, + npmAuthStatusSchema, + installPluginPayloadSchema, } from "./schemas.js"; // ============================================================================ @@ -198,6 +201,9 @@ export type provider_manifest = z.infer; export type plugin_document = z.infer; export type provider_document = z.infer; export type channel_routing_document = z.infer; +export type npm_auth_config = z.infer; +export type npm_auth_status = z.infer; +export type install_plugin_payload = z.infer; export interface KafkaConsumerState { consumer: import('kafkajs').Consumer | null; @@ -351,7 +357,8 @@ export type PluginSyncAction = | 'PROVIDER_UPSERTED' | 'PROVIDER_DELETED' | 'CHANNEL_ROUTING_UPDATED' - | 'SYSTEM_RELOAD'; + | 'SYSTEM_RELOAD' + | 'NPM_AUTH_UPDATED'; export interface PluginSyncPayload { plugin_name?: string; diff --git a/tests/integration/api/plugins.test.ts b/tests/integration/api/plugins.test.ts new file mode 100644 index 0000000..4fae9f3 --- /dev/null +++ b/tests/integration/api/plugins.test.ts @@ -0,0 +1,249 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import pluginsRouter from '../../../src/api/routes/plugins.routes.js'; +import { PluginManagerService, NpmAuthService } from '../../../src/plugins/index.js'; + +vi.mock('../../../src/plugins/index.js', () => ({ + PluginRegistry: { + isInitialized: vi.fn(() => true), + getPluginMetadata: vi.fn(() => ({ + channels: ['email'], + providers: { + email: [ + { + name: 'resend', + package: '@simplens/resend', + version: '1.0.0', + channel: 'email', + }, + ], + }, + counts: { channels: 1, providers: 1 }, + })), + }, + PluginManagerService: { + listInstalledPlugins: vi.fn(), + installPlugin: vi.fn(), + changePluginVersion: vi.fn(), + uninstallPlugin: vi.fn(), + }, + NpmAuthService: { + getAuthStatus: vi.fn(), + saveNpmAuth: vi.fn(), + deleteNpmAuth: vi.fn(), + }, +})); + +describe('Plugins API Integration', () => { + let app: express.Application; + + beforeEach(() => { + vi.clearAllMocks(); + app = express(); + app.use(express.json()); + app.use('/api/plugins', pluginsRouter); + }); + + describe('GET /api/plugins/npm-auth', () => { + it('should return npm auth status', async () => { + vi.mocked(NpmAuthService.getAuthStatus).mockResolvedValue({ + configured: true, + masked_token: 'npm_****cdef', + registry_url: 'https://registry.npmjs.org/', + }); + + const res = await request(app).get('/api/plugins/npm-auth'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + configured: true, + masked_token: 'npm_****cdef', + registry_url: 'https://registry.npmjs.org/', + }); + expect(NpmAuthService.getAuthStatus).toHaveBeenCalledTimes(1); + }); + }); + + describe('POST /api/plugins/npm-auth', () => { + it('should save npm auth configuration with valid payload', async () => { + vi.mocked(NpmAuthService.saveNpmAuth).mockResolvedValue({ + configured: true, + masked_token: 'npm_****7890', + registry_url: 'https://registry.npmjs.org/', + }); + + const res = await request(app) + .post('/api/plugins/npm-auth') + .send({ + token: 'npm_1234567890', + registry_url: 'https://registry.npmjs.org/', + }); + + expect(res.status).toBe(200); + expect(res.body.status.configured).toBe(true); + expect(NpmAuthService.saveNpmAuth).toHaveBeenCalledWith( + 'npm_1234567890', + 'https://registry.npmjs.org/' + ); + }); + + it('should reject invalid payload when token is empty', async () => { + const res = await request(app) + .post('/api/plugins/npm-auth') + .send({ + token: '', + }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Bad Request'); + expect(res.body.details).toBeDefined(); + }); + }); + + describe('DELETE /api/plugins/npm-auth', () => { + it('should delete npm auth configuration', async () => { + vi.mocked(NpmAuthService.deleteNpmAuth).mockResolvedValue(undefined); + + const res = await request(app).delete('/api/plugins/npm-auth'); + + expect(res.status).toBe(200); + expect(res.body.message).toContain('removed successfully'); + expect(NpmAuthService.deleteNpmAuth).toHaveBeenCalledTimes(1); + }); + }); + + describe('POST /api/plugins/install', () => { + it('should install plugin successfully without auth', async () => { + const mockPlugin = { + name: 'custom-email', + package: '@myorg/custom-email', + version: '1.0.0', + channel: 'email', + status: 'installed', + installed_at: new Date().toISOString(), + }; + vi.mocked(PluginManagerService.installPlugin).mockResolvedValue(mockPlugin as any); + + const res = await request(app) + .post('/api/plugins/install') + .send({ + package: '@myorg/custom-email', + version: '1.0.0', + }); + + expect(res.status).toBe(201); + expect(res.body.plugin).toEqual(mockPlugin); + expect(PluginManagerService.installPlugin).toHaveBeenCalledWith( + '@myorg/custom-email', + '1.0.0', + undefined + ); + }); + + it('should install private plugin with auth credentials provided', async () => { + const mockPlugin = { + name: 'private-slack', + package: '@enterprise/simplens-slack', + version: '2.0.0', + channel: 'slack', + status: 'installed', + installed_at: new Date().toISOString(), + }; + vi.mocked(PluginManagerService.installPlugin).mockResolvedValue(mockPlugin as any); + + const res = await request(app) + .post('/api/plugins/install') + .send({ + package: '@enterprise/simplens-slack', + version: '2.0.0', + auth: { + token: 'npm_secret_enterprise_token', + }, + }); + + expect(res.status).toBe(201); + expect(PluginManagerService.installPlugin).toHaveBeenCalledWith( + '@enterprise/simplens-slack', + '2.0.0', + { token: 'npm_secret_enterprise_token', save_token: true } + ); + }); + + it('should return 400 when package name is missing', async () => { + const res = await request(app) + .post('/api/plugins/install') + .send({ + version: '1.0.0', + }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Bad Request'); + expect(res.body.message).toContain('Invalid plugin installation payload'); + }); + + it('should return 400 when package is not a valid SimpleNS plugin', async () => { + vi.mocked(PluginManagerService.installPlugin).mockRejectedValue( + new Error("Package 'lodash' is not a valid SimpleNS plugin: Missing required provider methods: send, initialize") + ); + + const res = await request(app) + .post('/api/plugins/install') + .send({ + package: 'lodash', + }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Bad Request'); + expect(res.body.message).toContain('not a valid SimpleNS plugin'); + }); + + it('should return 400 when private package npm authentication fails', async () => { + vi.mocked(PluginManagerService.installPlugin).mockRejectedValue( + new Error("npm authentication failed (401/403) while installing '@private/pkg'. Configure npm auth token first.") + ); + + const res = await request(app) + .post('/api/plugins/install') + .send({ + package: '@private/pkg', + }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Bad Request'); + expect(res.body.message).toContain('npm authentication failed'); + }); + }); + + describe('GET /api/plugins/installed', () => { + it('should return list of installed plugins', async () => { + vi.mocked(PluginManagerService.listInstalledPlugins).mockResolvedValue([ + { + name: 'resend', + package: '@simplens/resend', + version: '1.0.0', + channel: 'email', + status: 'installed', + } as any, + ]); + + const res = await request(app).get('/api/plugins/installed'); + + expect(res.status).toBe(200); + expect(res.body.plugins).toHaveLength(1); + expect(res.body.plugins[0].package).toBe('@simplens/resend'); + }); + }); + + describe('DELETE /api/plugins/:package', () => { + it('should uninstall a plugin', async () => { + vi.mocked(PluginManagerService.uninstallPlugin).mockResolvedValue(undefined); + + const res = await request(app).delete('/api/plugins/@simplens/resend'); + + expect(res.status).toBe(200); + expect(res.body.message).toContain('uninstalled successfully'); + expect(PluginManagerService.uninstallPlugin).toHaveBeenCalledWith('@simplens/resend'); + }); + }); +}); diff --git a/tests/integration/workers/consumers/status.consumer.test.ts b/tests/integration/workers/consumers/status.consumer.test.ts index 23d5a46..1ed5b87 100644 --- a/tests/integration/workers/consumers/status.consumer.test.ts +++ b/tests/integration/workers/consumers/status.consumer.test.ts @@ -191,7 +191,7 @@ describe('Status Consumer', () => { expect.objectContaining({ status: 'delivered', }), - { new: true } + { returnDocument: 'after' } ); }); diff --git a/tests/unit/plugins/npm-auth.service.test.ts b/tests/unit/plugins/npm-auth.service.test.ts new file mode 100644 index 0000000..36416ff --- /dev/null +++ b/tests/unit/plugins/npm-auth.service.test.ts @@ -0,0 +1,139 @@ +/** + * Unit Tests for NpmAuthService + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { + NpmAuthService, + extractScope, + toNpmrcTokenLine, + NPM_AUTH_CONFIG_KEY, +} from '../../../src/plugins/services/npm-auth.service.js'; +import system_config_model from '../../../src/database/models/system-config.models.js'; +import * as keyManager from '../../../src/admin-alerts/key-manager.js'; +import { encrypt } from '../../../src/utils/encryption.utils.js'; +import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; +import * as fs from 'fs'; + +vi.mock('fs', async () => { + const actual = await vi.importActual('fs'); + return { + ...actual, + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), + existsSync: vi.fn().mockReturnValue(true), + }; +}); + +describe('NpmAuthService', () => { + const mockEncryptionKey = Buffer.alloc(32, 7); + + beforeEach(() => { + vi.restoreAllMocks(); + vi.spyOn(keyManager, 'getOrCreateEncryptionKey').mockResolvedValue(mockEncryptionKey); + vi.spyOn(PluginSyncService, 'publish').mockResolvedValue(undefined); + }); + + describe('extractScope', () => { + it('should extract scope from scoped package name', () => { + expect(extractScope('@simplens/resend')).toBe('@simplens'); + expect(extractScope('@myenterprise/custom-sms')).toBe('@myenterprise'); + expect(extractScope('@org-name/sub-pkg')).toBe('@org-name'); + }); + + it('should return undefined for unscoped package names', () => { + expect(extractScope('nodemailer-gmail')).toBeUndefined(); + expect(extractScope('custom-plugin')).toBeUndefined(); + }); + }); + + describe('toNpmrcTokenLine', () => { + it('should correctly format standard npmjs URL', () => { + const line = toNpmrcTokenLine('https://registry.npmjs.org/', 'npm_test123'); + expect(line).toBe('//registry.npmjs.org/:_authToken=npm_test123'); + }); + + it('should correctly format enterprise registry URL', () => { + const line = toNpmrcTokenLine('https://npm.enterprise.corp/', 'npm_corp_token'); + expect(line).toBe('//npm.enterprise.corp/:_authToken=npm_corp_token'); + }); + + it('should handle URLs without trailing slash', () => { + const line = toNpmrcTokenLine('https://npm.enterprise.corp', 'npm_corp_token'); + expect(line).toBe('//npm.enterprise.corp/:_authToken=npm_corp_token'); + }); + }); + + describe('getAuthStatus & getDecryptedToken', () => { + it('should return is_configured false when no token is stored and no ENV var exists', async () => { + delete process.env.NPM_TOKEN; + delete process.env.NPM_AUTH_TOKEN; + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue(null), + } as any); + + const status = await NpmAuthService.getAuthStatus(); + expect(status.is_configured).toBe(false); + expect(status.masked_token).toBeUndefined(); + }); + + it('should decrypt stored token and mask it for status display', async () => { + const plainToken = 'npm_1234567890abcdef123456'; + const encrypted = encrypt(plainToken, mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + encrypted_token: encrypted, + registry_url: 'https://registry.npmjs.org/', + }, + }), + } as any); + + const status = await NpmAuthService.getAuthStatus(); + expect(status.is_configured).toBe(true); + expect(status.registry_url).toBe('https://registry.npmjs.org/'); + expect(status.masked_token).toContain('****'); + expect(status.masked_token?.startsWith('npm_')).toBe(true); + expect(status.masked_token?.endsWith('3456')).toBe(true); + }); + }); + + describe('saveNpmAuth & deleteNpmAuth', () => { + it('should encrypt token, upsert in DB, and broadcast sync event', async () => { + const upsertSpy = vi.spyOn(system_config_model, 'findOneAndUpdate').mockResolvedValue({} as any); + vi.spyOn(NpmAuthService, 'syncNpmrc').mockResolvedValue(undefined); + vi.spyOn(NpmAuthService, 'getAuthStatus').mockResolvedValue({ + is_configured: true, + registry_url: 'https://npm.enterprise.corp/', + masked_token: 'npm_****1234', + }); + + const res = await NpmAuthService.saveNpmAuth('npm_my_secret_token', 'https://npm.enterprise.corp/'); + + expect(upsertSpy).toHaveBeenCalledWith( + { key: NPM_AUTH_CONFIG_KEY }, + expect.objectContaining({ + key: NPM_AUTH_CONFIG_KEY, + value: expect.objectContaining({ + registry_url: 'https://npm.enterprise.corp/', + }), + }), + { upsert: true, new: true } + ); + expect(PluginSyncService.publish).toHaveBeenCalledWith('NPM_AUTH_UPDATED', {}); + expect(res.is_configured).toBe(true); + }); + + it('should delete from DB and broadcast sync event', async () => { + const deleteSpy = vi.spyOn(system_config_model, 'deleteOne').mockResolvedValue({} as any); + vi.spyOn(NpmAuthService, 'syncNpmrc').mockResolvedValue(undefined); + + await NpmAuthService.deleteNpmAuth(); + + expect(deleteSpy).toHaveBeenCalledWith({ key: NPM_AUTH_CONFIG_KEY }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('NPM_AUTH_UPDATED', {}); + }); + }); +}); diff --git a/tests/unit/plugins/plugin-manager.service.test.ts b/tests/unit/plugins/plugin-manager.service.test.ts index 6cf5fe5..9be20c6 100644 --- a/tests/unit/plugins/plugin-manager.service.test.ts +++ b/tests/unit/plugins/plugin-manager.service.test.ts @@ -71,6 +71,20 @@ describe('PluginManagerService', () => { }); expect(result.version).toBe('2.1.0'); }); + + it('should roll back by uninstalling if manifest extraction/validation fails', async () => { + vi.spyOn(pluginFs, 'installNpmPackage').mockReturnValue(undefined); + const uninstallSpy = vi.spyOn(pluginFs, 'uninstallNpmPackage').mockReturnValue(undefined); + vi.spyOn(pluginFs, 'extractPackageManifest').mockRejectedValue( + new Error("Package 'lodash' is not a valid SimpleNS plugin.") + ); + + await expect( + PluginManagerService.installPlugin('lodash', 'latest') + ).rejects.toThrow("Package 'lodash' is not a valid SimpleNS plugin."); + + expect(uninstallSpy).toHaveBeenCalledWith('lodash'); + }); }); describe('changePluginVersion', () => { diff --git a/tests/unit/plugins/plugin-validation.test.ts b/tests/unit/plugins/plugin-validation.test.ts new file mode 100644 index 0000000..af18f7c --- /dev/null +++ b/tests/unit/plugins/plugin-validation.test.ts @@ -0,0 +1,85 @@ +/** + * Unit Tests for SimpleNSProvider Validation and Rollback + */ + +import { describe, it, expect, vi } from 'vitest'; +import { validateSimpleNSProvider } from '../../../src/plugins/loader/plugin-fs.js'; + +describe('validateSimpleNSProvider', () => { + const validProvider = { + manifest: { + name: '@simplens/mock', + displayName: 'Mock Provider', + version: '1.0.0', + channel: 'mock', + description: 'A mock provider for testing', + requiredCredentials: [], + }, + send: vi.fn(), + initialize: vi.fn(), + healthCheck: vi.fn(), + shutdown: vi.fn(), + getNotificationSchema: vi.fn(), + getRecipientSchema: vi.fn(), + getContentSchema: vi.fn(), + getRateLimitConfig: vi.fn(), + }; + + it('should accept a valid SimpleNSProvider implementation', () => { + const validated = validateSimpleNSProvider(validProvider, '@simplens/mock'); + expect(validated).toBeDefined(); + expect(validated.manifest.name).toBe('@simplens/mock'); + }); + + it('should reject a non-object or null export', () => { + expect(() => validateSimpleNSProvider(null, 'bad-pkg')).toThrow( + "Package 'bad-pkg' is not a valid SimpleNS plugin: default export must be a Provider object or class." + ); + expect(() => validateSimpleNSProvider(123, 'bad-pkg')).toThrow( + "Package 'bad-pkg' is not a valid SimpleNS plugin: default export must be a Provider object or class." + ); + }); + + it('should reject when manifest is missing', () => { + const missingManifest = { ...validProvider, manifest: undefined }; + expect(() => validateSimpleNSProvider(missingManifest, 'missing-manifest')).toThrow( + "Package 'missing-manifest' is not a valid SimpleNS plugin: missing 'manifest' definition." + ); + }); + + it('should reject when manifest fails schema validation (e.g. missing channel)', () => { + const invalidManifest = { + ...validProvider, + manifest: { + name: 'test', + // channel missing + displayName: 'Test', + description: 'desc', + version: '1.0.0', + requiredCredentials: [], + }, + }; + expect(() => validateSimpleNSProvider(invalidManifest, 'invalid-manifest')).toThrow( + "has an invalid plugin manifest" + ); + }); + + it('should reject arbitrary packages missing required provider methods (e.g. lodash / express)', () => { + const arbitraryPackage = { + default: () => ({ someUtility: true }), + manifest: { + name: 'arbitrary', + displayName: 'Arbitrary', + version: '1.0.0', + channel: 'email', + description: 'Not a real provider', + requiredCredentials: [], + }, + // missing send, initialize, healthCheck, etc. + }; + + expect(() => validateSimpleNSProvider(arbitraryPackage, 'lodash')).toThrow( + "Package 'lodash' is not a valid SimpleNS plugin. Missing required provider methods: send, initialize, healthCheck, shutdown, getNotificationSchema, getRecipientSchema, getContentSchema, getRateLimitConfig" + ); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 1d5fb31..5331216 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,9 +4,8 @@ "lib": ["ES2022"], "module": "NodeNext", "moduleResolution": "NodeNext", - "baseUrl": ".", "paths": { - "@src/*": ["src/*"] + "@src/*": ["./src/*"] }, "outDir": "./dist", "rootDir": "./src", From 3838f10c276f1b0101b685f6715a8c8eaaaf295e Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 20 Sep 2026 00:19:00 +0530 Subject: [PATCH 20/46] feat: enhance npm auth management with support for multiple registries and scopes - Updated API client to handle npm auth configurations with optional scope. - Modified types to include support for multiple npm registries and their statuses. - Enhanced the npm auth service to manage multiple tokens and registry configurations. - Updated controller methods to accommodate new parameters for saving and deleting npm auth configurations. - Improved tests to cover new functionality for multiple registries and scoped tokens. --- .../plugins/installed-plugins-tab.tsx | 172 +++++++-- dashboard/lib/api-client.ts | 5 +- dashboard/lib/types.ts | 12 +- src/api/controllers/plugins.controller.ts | 15 +- src/plugins/services/npm-auth.service.ts | 355 +++++++++++++++--- .../services/plugin-manager.service.ts | 7 +- src/types/schemas.ts | 28 +- src/types/types.ts | 4 + tests/integration/api/plugins.test.ts | 54 ++- tests/unit/plugins/npm-auth.service.test.ts | 274 ++++++++++++-- 10 files changed, 792 insertions(+), 134 deletions(-) diff --git a/dashboard/components/plugins/installed-plugins-tab.tsx b/dashboard/components/plugins/installed-plugins-tab.tsx index 3dbbafe..13bc004 100644 --- a/dashboard/components/plugins/installed-plugins-tab.tsx +++ b/dashboard/components/plugins/installed-plugins-tab.tsx @@ -31,6 +31,7 @@ import { import { Skeleton } from "@/components/ui/skeleton"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Download, Package, RefreshCw, Trash2, ArrowUpDown, Loader2, ExternalLink, Info, Search, Lock, ShieldCheck, Eye, EyeOff, Check } from "lucide-react"; import { toast } from "sonner"; @@ -50,11 +51,11 @@ export function InstalledPluginsTab() { const [customPkg, setCustomPkg] = useState(""); const [customVersion, setCustomVersion] = useState(""); const [isPrivate, setIsPrivate] = useState(false); - const [useSavedToken, setUseSavedToken] = useState(true); const [npmToken, setNpmToken] = useState(""); const [showToken, setShowToken] = useState(false); const [enterpriseRegistryUrl, setEnterpriseRegistryUrl] = useState(""); const [saveToken, setSaveToken] = useState(true); + const [overrideToken, setOverrideToken] = useState(false); const [isInstallingCustom, setIsInstallingCustom] = useState(false); const { data: npmAuth, mutate: mutateNpmAuth } = useSWR( @@ -62,6 +63,63 @@ export function InstalledPluginsTab() { () => pluginService.getNpmAuth() ); + const detectedScope = useMemo(() => { + const match = customPkg.trim().match(/^(@[a-zA-Z0-9~_.-]+)\//); + return match ? match[1].toLowerCase() : undefined; + }, [customPkg]); + + const matchedAuth = useMemo(() => { + if (!npmAuth?.is_configured) return null; + const regList = npmAuth.registries || []; + + // 1. Check scope match + if (detectedScope) { + const scopeMatch = regList.find( + (r) => r.scope?.toLowerCase() === detectedScope + ); + if (scopeMatch) return scopeMatch; + } + + // 2. Check registry URL match if entered + const regUrl = enterpriseRegistryUrl.trim(); + if (regUrl) { + try { + const inputUrl = regUrl.startsWith("http") ? regUrl : `https://${regUrl}`; + const inputHost = new URL(inputUrl).host.toLowerCase(); + const urlMatch = regList.find((r) => { + try { + return new URL(r.registry_url).host.toLowerCase() === inputHost; + } catch { + return false; + } + }); + if (urlMatch) return urlMatch; + } catch { + // invalid URL + } + } + + // 3. Check default npmjs match if no custom scope and no custom url + if (!detectedScope && !regUrl) { + const defaultMatch = regList.find( + (r) => !r.scope && r.registry_url.includes("registry.npmjs.org") + ); + if (defaultMatch) return defaultMatch; + } + + // 4. Fallback if single global/legacy token exists and neither custom scope nor custom url + if (!detectedScope && !regUrl && npmAuth.masked_token) { + return regList[0] || { + id: "default", + registry_url: npmAuth.default_registry || "https://registry.npmjs.org/", + masked_token: npmAuth.masked_token, + updated_at: "", + }; + } + + return null; + }, [npmAuth, detectedScope, enterpriseRegistryUrl]); + const { data: officialCatalog = [], isLoading: officialLoading, error: officialError } = useSWR( installModalOpen && catalogTab === "official" ? "plugin-catalog-official" : null, () => pluginService.listCatalog("official") @@ -114,15 +172,17 @@ export function InstalledPluginsTab() { }; if (isPrivate) { - const token = useSavedToken && npmAuth?.is_configured ? undefined : npmToken.trim(); - if (!useSavedToken && !token) { + const hasConfiguredAuth = matchedAuth && !overrideToken; + const token = hasConfiguredAuth ? undefined : npmToken.trim(); + if (!hasConfiguredAuth && !token) { toast.error("Please enter an npm authentication token for this private package"); return; } - if (token || enterpriseRegistryUrl.trim()) { + if (token || enterpriseRegistryUrl.trim() || detectedScope) { payload.auth = { token: token || undefined, registry_url: enterpriseRegistryUrl.trim() || undefined, + scope: detectedScope, save_token: saveToken, }; } @@ -140,6 +200,8 @@ export function InstalledPluginsTab() { setCustomPkg(""); setCustomVersion(""); setNpmToken(""); + setEnterpriseRegistryUrl(""); + setOverrideToken(false); setIsPrivate(false); setInstallModalOpen(false); } catch (err: unknown) { @@ -637,32 +699,86 @@ export function InstalledPluginsTab() { {isPrivate && (
- {npmAuth?.is_configured && ( -
-
+ {matchedAuth && !overrideToken ? ( +
+
- - Saved token active: {npmAuth.masked_token} - +
+
+ Configured: Using saved credentials for {matchedAuth.scope || matchedAuth.registry_url} +
+
+ Active Token: {matchedAuth.masked_token} +
+
- +
- )} - - {(!useSavedToken || !npmAuth?.is_configured) && ( + ) : (
+ {overrideToken && ( +
+ + Updating credentials for {detectedScope || "this registry"} + + +
+ )} +
- +
+ + + + + + +
+ + Token Requirements & Compatibility +
+
    +
  • + CI & Automation (No 2FA): SimpleNS runs installs in headless background processes. For npmjs.com, generate an Automation token (Classic token type "Automation") or a Granular Access Token configured with No 2FA prompt on install and Read-only permissions. +
    • +
  • + Scope Reusability: A single organization or user token covers all private packages under that scope (e.g. @my-org/*). You do not need a separate token for each package. +
    • +
  • + Enterprise Registries: For Verdaccio, Nexus, or Artifactory, enter an API key or service account token with read permissions. +
    • +
+ + +
+
setNpmToken(e.target.value)} - required={isPrivate && (!useSavedToken || !npmAuth?.is_configured)} + required={isPrivate && (!matchedAuth || overrideToken)} className="pr-10 font-mono text-xs" disabled={isInstallingCustom} />
diff --git a/dashboard/lib/api-client.ts b/dashboard/lib/api-client.ts index b3f5a1c..ad0504d 100644 --- a/dashboard/lib/api-client.ts +++ b/dashboard/lib/api-client.ts @@ -161,9 +161,10 @@ export const pluginService = { uninstall: (packageName: string): Promise<{ message: string }> => apiClient.delete(`/api/plugins/${encodeURIComponent(packageName)}`), getNpmAuth: (): Promise => apiClient.get('/api/plugins/npm-auth'), - saveNpmAuth: (payload: { token: string; registry_url?: string }): Promise<{ message: string; status: NpmAuthStatus }> => + saveNpmAuth: (payload: { token: string; registry_url?: string; scope?: string }): Promise<{ message: string; status: NpmAuthStatus }> => apiClient.post('/api/plugins/npm-auth', payload), - deleteNpmAuth: (): Promise<{ message: string }> => apiClient.delete('/api/plugins/npm-auth'), + deleteNpmAuth: (idOrScope?: string): Promise<{ message: string }> => + apiClient.delete('/api/plugins/npm-auth', { params: idOrScope ? { id: idOrScope } : undefined }), }; export const providerService = { diff --git a/dashboard/lib/types.ts b/dashboard/lib/types.ts index b483299..d0605d1 100644 --- a/dashboard/lib/types.ts +++ b/dashboard/lib/types.ts @@ -251,9 +251,18 @@ export interface PluginCatalogItem { versions?: string[]; } +export interface NpmRegistryStatus { + id: string; + scope?: string; + registry_url: string; + masked_token: string; + updated_at: string; +} + export interface NpmAuthStatus { is_configured: boolean; - registry_url?: string; + registries?: NpmRegistryStatus[]; + default_registry?: string; masked_token?: string; } @@ -263,6 +272,7 @@ export interface InstallPluginPayload { auth?: { token?: string; registry_url?: string; + scope?: string; save_token?: boolean; }; } diff --git a/src/api/controllers/plugins.controller.ts b/src/api/controllers/plugins.controller.ts index fc19559..ce6d58c 100644 --- a/src/api/controllers/plugins.controller.ts +++ b/src/api/controllers/plugins.controller.ts @@ -229,8 +229,8 @@ export const saveNpmAuth = async (req: Request, res: Response): Promise => return; } - const { token, registry_url } = validation.data; - const status = await NpmAuthService.saveNpmAuth(token, registry_url); + const { token, registry_url, scope } = validation.data; + const status = await NpmAuthService.saveNpmAuth(token, registry_url, scope); res.status(200).json({ message: 'npm auth configuration saved successfully', status, @@ -247,13 +247,16 @@ export const saveNpmAuth = async (req: Request, res: Response): Promise => /** * DELETE /api/plugins/npm-auth - * Delete saved npm auth configuration + * Delete saved npm auth configuration (specific registry or all) */ -export const deleteNpmAuth = async (_req: Request, res: Response): Promise => { +export const deleteNpmAuth = async (req: Request, res: Response): Promise => { try { - await NpmAuthService.deleteNpmAuth(); + const target = (req.query.id as string) || (req.query.scope as string) || (req.body?.id as string); + await NpmAuthService.deleteNpmAuth(target); res.json({ - message: 'npm auth configuration removed successfully', + message: target + ? `npm auth configuration for '${target}' removed successfully` + : 'npm auth configuration removed successfully', }); } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); diff --git a/src/plugins/services/npm-auth.service.ts b/src/plugins/services/npm-auth.service.ts index 28e58d8..ae34cd5 100644 --- a/src/plugins/services/npm-auth.service.ts +++ b/src/plugins/services/npm-auth.service.ts @@ -1,7 +1,7 @@ /** * NPM Authentication & Registry Configuration Service * - * Manages npm access tokens and enterprise registry configurations, + * Manages multiple npm access tokens and enterprise registry configurations, * encrypted storage in MongoDB, and synchronization of .plugins/.npmrc. */ @@ -12,15 +12,37 @@ import { getOrCreateEncryptionKey } from '@src/admin-alerts/key-manager.js'; import { encrypt, decrypt } from '@src/utils/encryption.utils.js'; import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; import { PLUGINS_DIR, initPluginsDir } from '@src/plugins/loader/plugin-fs.js'; -import type { encrypted_config, npm_auth_status } from '@src/types/types.js'; +import type { + encrypted_config, + npm_auth_status, + npm_registry_status, +} from '@src/types/types.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; export const NPM_AUTH_CONFIG_KEY = 'npm_auth_config'; export const NPMRC_PATH = join(PLUGINS_DIR, '.npmrc'); -interface StoredNpmAuthConfig { +interface StoredRegistryEntry { + id: string; + scope?: string; + registry_url: string; encrypted_token: encrypted_config; + updated_at: string; +} + +interface StoredNpmAuthConfig { + registries?: StoredRegistryEntry[]; + // Legacy fields for backwards compatibility + encrypted_token?: encrypted_config; + registry_url?: string; +} + +export interface DecryptedRegistryEntry { + id: string; + scope?: string; registry_url: string; + token: string; + updated_at: string; } /** @@ -29,7 +51,47 @@ interface StoredNpmAuthConfig { */ export function extractScope(packageName: string): string | undefined { const match = packageName.trim().match(/^(@[a-zA-Z0-9~_.-]+)\//); - return match ? match[1] : undefined; + return match ? match[1].toLowerCase() : undefined; +} + +/** + * Normalize npm registry URL to ensure valid URL with trailing slash + */ +export function normalizeRegistryUrl(registryUrl?: string): string { + if (!registryUrl || !registryUrl.trim()) { + return 'https://registry.npmjs.org/'; + } + let trimmed = registryUrl.trim(); + if (!trimmed.endsWith('/')) { + trimmed += '/'; + } + return trimmed; +} + +/** + * Generate a consistent identifier for a registry entry + */ +export function generateRegistryId(scope?: string, registryUrl?: string): string { + if (scope) { + return `scope:${scope.trim().toLowerCase()}`; + } + const normalized = normalizeRegistryUrl(registryUrl); + if (!normalized.includes('registry.npmjs.org')) { + try { + return `url:${new URL(normalized).host.toLowerCase()}`; + } catch { + // fallback + } + } + return 'default'; +} + +/** + * Mask an authentication token for safe public/UI presentation + */ +export function maskToken(token: string): string { + const t = token.trim(); + return t.length > 8 ? `${t.slice(0, 4)}****${t.slice(-4)}` : '****'; } /** @@ -51,74 +113,220 @@ export function toNpmrcTokenLine(registryUrl: string, token: string): string { export class NpmAuthService { /** - * Retrieve active npm auth credentials from MongoDB or process.env + * Retrieve all stored registry entries, migrating legacy single-token documents if present */ - public static async getDecryptedToken(): Promise<{ token: string; registry_url: string } | null> { + private static async getRawStoredEntries(): Promise { try { const doc = await system_config_model.findOne({ key: NPM_AUTH_CONFIG_KEY }).lean(); - if (doc?.value) { - const stored = doc.value as StoredNpmAuthConfig; - if (stored.encrypted_token) { - const key = await getOrCreateEncryptionKey(); - const token = decrypt(stored.encrypted_token, key); - return { - token, - registry_url: stored.registry_url || 'https://registry.npmjs.org/', - }; - } + if (!doc?.value) { + return []; } + + const stored = doc.value as StoredNpmAuthConfig; + const entries: StoredRegistryEntry[] = []; + + if (Array.isArray(stored.registries) && stored.registries.length > 0) { + entries.push(...stored.registries); + } else if (stored.encrypted_token) { + // Migrate legacy single-token document + const regUrl = normalizeRegistryUrl(stored.registry_url); + entries.push({ + id: generateRegistryId(undefined, regUrl), + registry_url: regUrl, + encrypted_token: stored.encrypted_token, + updated_at: new Date().toISOString(), + }); + } + + return entries; } catch (err) { - logger.error('Failed to decrypt stored npm token from MongoDB:', err); + logger.error('Failed to read npm registries from MongoDB:', err); + return []; } + } - // Fall back to environment variable if present + /** + * Retrieve all decrypted registry configurations from MongoDB + environment variables + */ + public static async getDecryptedRegistries(): Promise { + const entries = await this.getRawStoredEntries(); + const result: DecryptedRegistryEntry[] = []; + + if (entries.length > 0) { + try { + const key = await getOrCreateEncryptionKey(); + for (const entry of entries) { + try { + const token = decrypt(entry.encrypted_token, key); + result.push({ + id: entry.id, + scope: entry.scope, + registry_url: normalizeRegistryUrl(entry.registry_url), + token, + updated_at: entry.updated_at, + }); + } catch (decErr) { + logger.error(`Failed to decrypt token for registry ${entry.id}:`, decErr); + } + } + } catch (keyErr) { + logger.error('Failed to get encryption key for npm registries:', keyErr); + } + } + + // Include environment variable fallback if not already present const envToken = process.env.NPM_TOKEN || process.env.NPM_AUTH_TOKEN; if (envToken) { + const envRegUrl = normalizeRegistryUrl(process.env.NPM_REGISTRY_URL); + const exists = result.some(r => r.registry_url === envRegUrl && !r.scope); + if (!exists) { + result.push({ + id: 'env-default', + registry_url: envRegUrl, + token: envToken, + updated_at: new Date().toISOString(), + }); + } + } + + return result; + } + + /** + * Retrieve active npm auth credentials (backwards-compatible helper) + */ + public static async getDecryptedToken(): Promise<{ token: string; registry_url: string } | null> { + const registries = await this.getDecryptedRegistries(); + if (registries.length > 0) { + const primary = registries[0]; return { - token: envToken, - registry_url: process.env.NPM_REGISTRY_URL || 'https://registry.npmjs.org/', + token: primary.token, + registry_url: primary.registry_url, }; } - return null; } /** - * Get public status of configured npm auth (with masked token) + * Find matching credentials for a given package name and/or target registry URL + */ + public static async findMatchingAuth( + packageName: string, + registryUrl?: string + ): Promise { + const registries = await this.getDecryptedRegistries(); + if (registries.length === 0) { + return null; + } + + const scope = extractScope(packageName); + const targetUrl = registryUrl ? normalizeRegistryUrl(registryUrl) : undefined; + + // 1. Try matching by package scope (e.g. "@myenterprise") + if (scope) { + const scopeMatch = registries.find(r => r.scope?.toLowerCase() === scope); + if (scopeMatch) { + return scopeMatch; + } + } + + // 2. Try matching by target registry URL host + if (targetUrl) { + try { + const targetHost = new URL(targetUrl).host.toLowerCase(); + const urlMatch = registries.find(r => { + try { + return new URL(r.registry_url).host.toLowerCase() === targetHost; + } catch { + return false; + } + }); + if (urlMatch) { + return urlMatch; + } + } catch { + // invalid URL ignore + } + } + + // 3. Fallback: If target is default npmjs or unspecifed, find default/global registry + const defaultMatch = registries.find(r => !r.scope && r.registry_url.includes('registry.npmjs.org')); + if (defaultMatch) { + return defaultMatch; + } + + // 4. Return first registry if no specific match and it's a global entry + const anyGlobal = registries.find(r => !r.scope); + return anyGlobal || null; + } + + /** + * Get public status of all configured registries (with masked tokens) */ public static async getAuthStatus(): Promise { - const auth = await this.getDecryptedToken(); - if (!auth || !auth.token) { + const registries = await this.getDecryptedRegistries(); + if (registries.length === 0) { return { is_configured: false, + registries: [], }; } - const t = auth.token; - const masked = t.length > 8 - ? `${t.slice(0, 4)}****${t.slice(-4)}` - : '****'; + const registryStatuses: npm_registry_status[] = registries.map(r => ({ + id: r.id, + scope: r.scope, + registry_url: r.registry_url, + masked_token: maskToken(r.token), + updated_at: r.updated_at, + })); + const primary = registries[0]; return { is_configured: true, - registry_url: auth.registry_url, - masked_token: masked, + registries: registryStatuses, + default_registry: primary.registry_url, + masked_token: maskToken(primary.token), }; } /** - * Save and encrypt npm auth configuration in MongoDB, sync .npmrc, and broadcast + * Save and encrypt npm auth configuration for a scope or registry in MongoDB */ - public static async saveNpmAuth(token: string, registryUrl?: string): Promise { + public static async saveNpmAuth( + token: string, + registryUrl?: string, + scope?: string + ): Promise { const cleanToken = token.trim(); - const cleanRegistry = registryUrl?.trim() || 'https://registry.npmjs.org/'; + const cleanRegistry = normalizeRegistryUrl(registryUrl); + const cleanScope = scope?.trim() ? (scope.startsWith('@') ? scope.trim().toLowerCase() : `@${scope.trim().toLowerCase()}`) : undefined; + const entryId = generateRegistryId(cleanScope, cleanRegistry); const key = await getOrCreateEncryptionKey(); const encryptedToken = encrypt(cleanToken, key); - const storedValue: StoredNpmAuthConfig = { - encrypted_token: encryptedToken, + const currentEntries = await this.getRawStoredEntries(); + const existingIndex = currentEntries.findIndex(e => e.id === entryId || (cleanScope && e.scope === cleanScope)); + + const newEntry: StoredRegistryEntry = { + id: entryId, + scope: cleanScope, registry_url: cleanRegistry, + encrypted_token: encryptedToken, + updated_at: new Date().toISOString(), + }; + + if (existingIndex >= 0) { + currentEntries[existingIndex] = newEntry; + } else { + currentEntries.push(newEntry); + } + + const storedValue: StoredNpmAuthConfig = { + registries: currentEntries, + // Clear legacy single fields to prevent ambiguity + encrypted_token: undefined, + registry_url: undefined, }; await system_config_model.findOneAndUpdate( @@ -130,22 +338,44 @@ export class NpmAuthService { await this.syncNpmrc(); await PluginSyncService.publish('NPM_AUTH_UPDATED', {}); - logger.success(`npm auth configuration saved and broadcasted successfully for ${cleanRegistry}`); + logger.success(`npm auth configuration saved and broadcasted for ${cleanScope || cleanRegistry} (${entryId})`); return this.getAuthStatus(); } /** - * Delete saved npm auth configuration, sync .npmrc, and broadcast + * Delete saved npm auth configuration by ID, scope, or delete all */ - public static async deleteNpmAuth(): Promise { - await system_config_model.deleteOne({ key: NPM_AUTH_CONFIG_KEY }); + public static async deleteNpmAuth(idOrScope?: string): Promise { + if (!idOrScope || idOrScope === 'all') { + await system_config_model.deleteOne({ key: NPM_AUTH_CONFIG_KEY }); + logger.info('All npm auth configurations removed'); + } else { + const currentEntries = await this.getRawStoredEntries(); + const target = idOrScope.trim().toLowerCase(); + const filtered = currentEntries.filter( + e => e.id.toLowerCase() !== target && e.scope?.toLowerCase() !== target && e.registry_url.toLowerCase() !== target + ); + + const storedValue: StoredNpmAuthConfig = { + registries: filtered, + encrypted_token: undefined, + registry_url: undefined, + }; + + await system_config_model.findOneAndUpdate( + { key: NPM_AUTH_CONFIG_KEY }, + { key: NPM_AUTH_CONFIG_KEY, value: storedValue }, + { upsert: true, new: true } + ); + logger.info(`Removed npm auth configuration for ${idOrScope}`); + } + await this.syncNpmrc(); await PluginSyncService.publish('NPM_AUTH_UPDATED', {}); - logger.info('npm auth configuration removed and broadcasted'); } /** - * Generate and write .plugins/.npmrc + * Generate and write .plugins/.npmrc containing all scoped and host credentials * Supports optional inline scope/registry/token for one-off private installs */ public static async syncNpmrc(temporaryOverride?: { @@ -160,32 +390,53 @@ export class NpmAuthService { 'package-lock=false', ]; - const activeAuth = await this.getDecryptedToken(); + const registries = await this.getDecryptedRegistries(); + const writtenHosts = new Set(); + const writtenScopes = new Set(); + + // 1. Process all persistent registries + for (const reg of registries) { + if (reg.scope && !writtenScopes.has(reg.scope)) { + if (!reg.registry_url.includes('registry.npmjs.org')) { + lines.push(`${reg.scope}:registry=${reg.registry_url}`); + } + writtenScopes.has(reg.scope); + } - // 1. Configured persistent auth - if (activeAuth && activeAuth.token) { - const isCustomRegistry = activeAuth.registry_url && !activeAuth.registry_url.includes('registry.npmjs.org'); - if (isCustomRegistry) { - lines.push(`registry=${activeAuth.registry_url}`); + const hostLine = toNpmrcTokenLine(reg.registry_url, reg.token); + if (!writtenHosts.has(hostLine)) { + lines.push(hostLine); + writtenHosts.add(hostLine); + } + + // If global custom registry with no scope + if (!reg.scope && !reg.registry_url.includes('registry.npmjs.org')) { + lines.push(`registry=${reg.registry_url}`); } - lines.push(toNpmrcTokenLine(activeAuth.registry_url, activeAuth.token)); } // 2. Temporary/Inline override (e.g. for a specific scoped install) if (temporaryOverride?.token) { - const targetRegistry = temporaryOverride.registryUrl || 'https://registry.npmjs.org/'; + const targetRegistry = normalizeRegistryUrl(temporaryOverride.registryUrl); if (temporaryOverride.scope) { - lines.push(`${temporaryOverride.scope}:registry=${targetRegistry}`); + const cleanScope = temporaryOverride.scope.startsWith('@') + ? temporaryOverride.scope.toLowerCase() + : `@${temporaryOverride.scope.toLowerCase()}`; + lines.push(`${cleanScope}:registry=${targetRegistry}`); } else if (!targetRegistry.includes('registry.npmjs.org')) { lines.push(`registry=${targetRegistry}`); } - lines.push(toNpmrcTokenLine(targetRegistry, temporaryOverride.token)); + const overrideHostLine = toNpmrcTokenLine(targetRegistry, temporaryOverride.token); + if (!writtenHosts.has(overrideHostLine)) { + lines.push(overrideHostLine); + writtenHosts.add(overrideHostLine); + } } const content = lines.join('\n') + '\n'; try { writeFileSync(NPMRC_PATH, content, { encoding: 'utf-8', mode: 0o600 }); - logger.debug(`Synchronized .npmrc in ${PLUGINS_DIR}`); + logger.debug(`Synchronized .npmrc in ${PLUGINS_DIR} with ${registries.length} registries`); } catch (err) { logger.error('Failed to write .npmrc:', err); } diff --git a/src/plugins/services/plugin-manager.service.ts b/src/plugins/services/plugin-manager.service.ts index fe0ebad..ada8e4e 100644 --- a/src/plugins/services/plugin-manager.service.ts +++ b/src/plugins/services/plugin-manager.service.ts @@ -86,13 +86,14 @@ export class PluginManagerService { version?: string, auth?: install_plugin_payload['auth'] ): Promise { + const scope = extractScope(packageName); if (auth?.token) { - const scope = extractScope(packageName); + const targetScope = auth.scope || scope; if (auth.save_token !== false) { - await NpmAuthService.saveNpmAuth(auth.token, auth.registry_url); + await NpmAuthService.saveNpmAuth(auth.token, auth.registry_url, targetScope); } else { await NpmAuthService.syncNpmrc({ - scope, + scope: targetScope, registryUrl: auth.registry_url, token: auth.token, }); diff --git a/src/types/schemas.ts b/src/types/schemas.ts index 42f6cbe..9c0833e 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -554,20 +554,43 @@ export const adminLoginSchema = z.object({ password: z.string().min(1, "Password is required"), }); +/** + * npm registry configuration schema for individual custom registries/scopes + */ +export const npmRegistryConfigSchema = z.object({ + id: z.string().optional(), + scope: z.string().trim().optional(), + registry_url: z.string().url("Invalid registry URL").default("https://registry.npmjs.org/"), + token: z.string().min(1, "npm token cannot be empty"), +}); + +/** + * npm registry status item for public/dashboard display + */ +export const npmRegistryStatusSchema = z.object({ + id: z.string(), + scope: z.string().optional(), + registry_url: z.string(), + masked_token: z.string(), + updated_at: z.string(), +}); + /** * npm registry authentication config schema */ export const npmAuthConfigSchema = z.object({ token: z.string().min(1, "npm token cannot be empty"), registry_url: z.string().url("Invalid registry URL").default("https://registry.npmjs.org/"), + scope: z.string().trim().optional(), }); /** - * npm registry auth status schema (safe to return to UI with masked token) + * npm registry auth status schema (safe to return to UI with masked token and registry list) */ export const npmAuthStatusSchema = z.object({ is_configured: z.boolean(), - registry_url: z.string().optional(), + registries: z.array(npmRegistryStatusSchema).default([]), + default_registry: z.string().optional(), masked_token: z.string().optional(), }); @@ -580,6 +603,7 @@ export const installPluginPayloadSchema = z.object({ auth: z.object({ token: z.string().optional(), registry_url: z.string().url("Invalid registry URL").optional(), + scope: z.string().optional(), save_token: z.boolean().default(true), }).optional(), }); diff --git a/src/types/types.ts b/src/types/types.ts index fbf1324..f692be8 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -132,6 +132,8 @@ import { partialOperationalSettingsSchema, adminSetupSchema, adminLoginSchema, + npmRegistryConfigSchema, + npmRegistryStatusSchema, npmAuthConfigSchema, npmAuthStatusSchema, installPluginPayloadSchema, @@ -201,6 +203,8 @@ export type provider_manifest = z.infer; export type plugin_document = z.infer; export type provider_document = z.infer; export type channel_routing_document = z.infer; +export type npm_registry_config = z.infer; +export type npm_registry_status = z.infer; export type npm_auth_config = z.infer; export type npm_auth_status = z.infer; export type install_plugin_payload = z.infer; diff --git a/tests/integration/api/plugins.test.ts b/tests/integration/api/plugins.test.ts index 4fae9f3..43c5179 100644 --- a/tests/integration/api/plugins.test.ts +++ b/tests/integration/api/plugins.test.ts @@ -46,45 +46,55 @@ describe('Plugins API Integration', () => { }); describe('GET /api/plugins/npm-auth', () => { - it('should return npm auth status', async () => { + it('should return npm auth status with registries list', async () => { vi.mocked(NpmAuthService.getAuthStatus).mockResolvedValue({ - configured: true, - masked_token: 'npm_****cdef', - registry_url: 'https://registry.npmjs.org/', + is_configured: true, + registries: [ + { + id: 'scope:@corp-a', + scope: '@corp-a', + registry_url: 'https://npm.corp-a.com/', + masked_token: 'npm_****1234', + updated_at: new Date().toISOString(), + }, + ], + masked_token: 'npm_****1234', + default_registry: 'https://registry.npmjs.org/', }); const res = await request(app).get('/api/plugins/npm-auth'); expect(res.status).toBe(200); - expect(res.body).toEqual({ - configured: true, - masked_token: 'npm_****cdef', - registry_url: 'https://registry.npmjs.org/', - }); + expect(res.body.is_configured).toBe(true); + expect(res.body.registries).toHaveLength(1); + expect(res.body.registries[0].scope).toBe('@corp-a'); expect(NpmAuthService.getAuthStatus).toHaveBeenCalledTimes(1); }); }); describe('POST /api/plugins/npm-auth', () => { - it('should save npm auth configuration with valid payload', async () => { + it('should save npm auth configuration with scope', async () => { vi.mocked(NpmAuthService.saveNpmAuth).mockResolvedValue({ - configured: true, + is_configured: true, + registries: [], masked_token: 'npm_****7890', - registry_url: 'https://registry.npmjs.org/', + default_registry: 'https://npm.corp.internal/', }); const res = await request(app) .post('/api/plugins/npm-auth') .send({ token: 'npm_1234567890', - registry_url: 'https://registry.npmjs.org/', + registry_url: 'https://npm.corp.internal/', + scope: '@myenterprise', }); expect(res.status).toBe(200); - expect(res.body.status.configured).toBe(true); + expect(res.body.status.is_configured).toBe(true); expect(NpmAuthService.saveNpmAuth).toHaveBeenCalledWith( 'npm_1234567890', - 'https://registry.npmjs.org/' + 'https://npm.corp.internal/', + '@myenterprise' ); }); @@ -102,14 +112,24 @@ describe('Plugins API Integration', () => { }); describe('DELETE /api/plugins/npm-auth', () => { - it('should delete npm auth configuration', async () => { + it('should delete npm auth configuration for a specific registry id', async () => { + vi.mocked(NpmAuthService.deleteNpmAuth).mockResolvedValue(undefined); + + const res = await request(app).delete('/api/plugins/npm-auth?id=scope:@corp-a'); + + expect(res.status).toBe(200); + expect(res.body.message).toContain('scope:@corp-a'); + expect(NpmAuthService.deleteNpmAuth).toHaveBeenCalledWith('scope:@corp-a'); + }); + + it('should delete all npm auth configurations when no target specified', async () => { vi.mocked(NpmAuthService.deleteNpmAuth).mockResolvedValue(undefined); const res = await request(app).delete('/api/plugins/npm-auth'); expect(res.status).toBe(200); expect(res.body.message).toContain('removed successfully'); - expect(NpmAuthService.deleteNpmAuth).toHaveBeenCalledTimes(1); + expect(NpmAuthService.deleteNpmAuth).toHaveBeenCalledWith(undefined); }); }); diff --git a/tests/unit/plugins/npm-auth.service.test.ts b/tests/unit/plugins/npm-auth.service.test.ts index 36416ff..829d108 100644 --- a/tests/unit/plugins/npm-auth.service.test.ts +++ b/tests/unit/plugins/npm-auth.service.test.ts @@ -7,7 +7,11 @@ import { NpmAuthService, extractScope, toNpmrcTokenLine, + normalizeRegistryUrl, + generateRegistryId, + maskToken, NPM_AUTH_CONFIG_KEY, + NPMRC_PATH, } from '../../../src/plugins/services/npm-auth.service.js'; import system_config_model from '../../../src/database/models/system-config.models.js'; import * as keyManager from '../../../src/admin-alerts/key-manager.js'; @@ -35,9 +39,9 @@ describe('NpmAuthService', () => { }); describe('extractScope', () => { - it('should extract scope from scoped package name', () => { + it('should extract scope from scoped package name and lowercase it', () => { expect(extractScope('@simplens/resend')).toBe('@simplens'); - expect(extractScope('@myenterprise/custom-sms')).toBe('@myenterprise'); + expect(extractScope('@MyEnterprise/custom-sms')).toBe('@myenterprise'); expect(extractScope('@org-name/sub-pkg')).toBe('@org-name'); }); @@ -64,8 +68,26 @@ describe('NpmAuthService', () => { }); }); - describe('getAuthStatus & getDecryptedToken', () => { - it('should return is_configured false when no token is stored and no ENV var exists', async () => { + describe('normalizeRegistryUrl & generateRegistryId & maskToken', () => { + it('should normalize URLs with trailing slash', () => { + expect(normalizeRegistryUrl('https://npm.corp.com')).toBe('https://npm.corp.com/'); + expect(normalizeRegistryUrl('')).toBe('https://registry.npmjs.org/'); + }); + + it('should generate consistent registry identifiers', () => { + expect(generateRegistryId('@myorg')).toBe('scope:@myorg'); + expect(generateRegistryId(undefined, 'https://npm.corp.com/')).toBe('url:npm.corp.com'); + expect(generateRegistryId(undefined, 'https://registry.npmjs.org/')).toBe('default'); + }); + + it('should mask tokens properly', () => { + expect(maskToken('npm_1234567890abcdef')).toBe('npm_****cdef'); + expect(maskToken('short')).toBe('****'); + }); + }); + + describe('getAuthStatus & getDecryptedRegistries', () => { + it('should return is_configured false when no registries exist and no ENV var exists', async () => { delete process.env.NPM_TOKEN; delete process.env.NPM_AUTH_TOKEN; vi.spyOn(system_config_model, 'findOne').mockReturnValue({ @@ -74,10 +96,10 @@ describe('NpmAuthService', () => { const status = await NpmAuthService.getAuthStatus(); expect(status.is_configured).toBe(false); - expect(status.masked_token).toBeUndefined(); + expect(status.registries).toHaveLength(0); }); - it('should decrypt stored token and mask it for status display', async () => { + it('should migrate legacy single-token document and return status with masked token', async () => { const plainToken = 'npm_1234567890abcdef123456'; const encrypted = encrypt(plainToken, mockEncryptionKey); @@ -93,47 +115,253 @@ describe('NpmAuthService', () => { const status = await NpmAuthService.getAuthStatus(); expect(status.is_configured).toBe(true); - expect(status.registry_url).toBe('https://registry.npmjs.org/'); - expect(status.masked_token).toContain('****'); - expect(status.masked_token?.startsWith('npm_')).toBe(true); - expect(status.masked_token?.endsWith('3456')).toBe(true); + expect(status.registries).toHaveLength(1); + expect(status.registries[0].masked_token).toContain('****'); + expect(status.registries[0].masked_token.startsWith('npm_')).toBe(true); + }); + + it('should return multiple configured registries with masked tokens', async () => { + const tokenA = encrypt('npm_corp_aaa_token_1234', mockEncryptionKey); + const tokenB = encrypt('npm_corp_bbb_token_5678', mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [ + { + id: 'scope:@corp-a', + scope: '@corp-a', + registry_url: 'https://npm.corp-a.com/', + encrypted_token: tokenA, + updated_at: new Date().toISOString(), + }, + { + id: 'scope:@corp-b', + scope: '@corp-b', + registry_url: 'https://npm.corp-b.com/', + encrypted_token: tokenB, + updated_at: new Date().toISOString(), + }, + ], + }, + }), + } as any); + + const status = await NpmAuthService.getAuthStatus(); + expect(status.is_configured).toBe(true); + expect(status.registries).toHaveLength(2); + expect(status.registries[0].scope).toBe('@corp-a'); + expect(status.registries[1].scope).toBe('@corp-b'); + expect(status.registries[0].masked_token).toBe('npm_****1234'); + expect(status.registries[1].masked_token).toBe('npm_****5678'); + }); + }); + + describe('findMatchingAuth', () => { + it('should match by scope when package name matches a registered scope', async () => { + const tokenA = encrypt('token_for_corp_a', mockEncryptionKey); + const tokenB = encrypt('token_for_corp_b', mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [ + { + id: 'scope:@corp-a', + scope: '@corp-a', + registry_url: 'https://npm.corp-a.com/', + encrypted_token: tokenA, + updated_at: new Date().toISOString(), + }, + { + id: 'scope:@corp-b', + scope: '@corp-b', + registry_url: 'https://npm.corp-b.com/', + encrypted_token: tokenB, + updated_at: new Date().toISOString(), + }, + ], + }, + }), + } as any); + + const match = await NpmAuthService.findMatchingAuth('@corp-a/email-provider'); + expect(match).not.toBeNull(); + expect(match?.scope).toBe('@corp-a'); + expect(match?.token).toBe('token_for_corp_a'); + expect(match?.registry_url).toBe('https://npm.corp-a.com/'); + }); + + it('should match by registry host when custom registry URL is specified', async () => { + const tokenCustom = encrypt('token_for_nexus', mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [ + { + id: 'url:nexus.internal.com', + registry_url: 'https://nexus.internal.com/repository/npm/', + encrypted_token: tokenCustom, + updated_at: new Date().toISOString(), + }, + ], + }, + }), + } as any); + + const match = await NpmAuthService.findMatchingAuth('unscoped-pkg', 'https://nexus.internal.com/repository/npm/'); + expect(match).not.toBeNull(); + expect(match?.token).toBe('token_for_nexus'); + }); + + it('should return null when no credentials match', async () => { + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue(null), + } as any); + + const match = await NpmAuthService.findMatchingAuth('@unregistered/package'); + expect(match).toBeNull(); }); }); describe('saveNpmAuth & deleteNpmAuth', () => { - it('should encrypt token, upsert in DB, and broadcast sync event', async () => { + it('should append or update registry entry in MongoDB and broadcast event', async () => { const upsertSpy = vi.spyOn(system_config_model, 'findOneAndUpdate').mockResolvedValue({} as any); + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [], + }, + }), + } as any); vi.spyOn(NpmAuthService, 'syncNpmrc').mockResolvedValue(undefined); - vi.spyOn(NpmAuthService, 'getAuthStatus').mockResolvedValue({ - is_configured: true, - registry_url: 'https://npm.enterprise.corp/', - masked_token: 'npm_****1234', - }); - const res = await NpmAuthService.saveNpmAuth('npm_my_secret_token', 'https://npm.enterprise.corp/'); + await NpmAuthService.saveNpmAuth('npm_enterprise_token', 'https://npm.enterprise.corp/', '@myorg'); expect(upsertSpy).toHaveBeenCalledWith( { key: NPM_AUTH_CONFIG_KEY }, expect.objectContaining({ key: NPM_AUTH_CONFIG_KEY, value: expect.objectContaining({ - registry_url: 'https://npm.enterprise.corp/', + registries: expect.arrayContaining([ + expect.objectContaining({ + id: 'scope:@myorg', + scope: '@myorg', + registry_url: 'https://npm.enterprise.corp/', + }), + ]), }), }), { upsert: true, new: true } ); expect(PluginSyncService.publish).toHaveBeenCalledWith('NPM_AUTH_UPDATED', {}); - expect(res.is_configured).toBe(true); }); - it('should delete from DB and broadcast sync event', async () => { - const deleteSpy = vi.spyOn(system_config_model, 'deleteOne').mockResolvedValue({} as any); + it('should delete specific registry by scope or id', async () => { + const upsertSpy = vi.spyOn(system_config_model, 'findOneAndUpdate').mockResolvedValue({} as any); + const tokenA = encrypt('tokenA', mockEncryptionKey); + const tokenB = encrypt('tokenB', mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [ + { + id: 'scope:@corp-a', + scope: '@corp-a', + registry_url: 'https://npm.corp-a.com/', + encrypted_token: tokenA, + updated_at: new Date().toISOString(), + }, + { + id: 'scope:@corp-b', + scope: '@corp-b', + registry_url: 'https://npm.corp-b.com/', + encrypted_token: tokenB, + updated_at: new Date().toISOString(), + }, + ], + }, + }), + } as any); vi.spyOn(NpmAuthService, 'syncNpmrc').mockResolvedValue(undefined); - await NpmAuthService.deleteNpmAuth(); + await NpmAuthService.deleteNpmAuth('@corp-a'); - expect(deleteSpy).toHaveBeenCalledWith({ key: NPM_AUTH_CONFIG_KEY }); + expect(upsertSpy).toHaveBeenCalledWith( + { key: NPM_AUTH_CONFIG_KEY }, + expect.objectContaining({ + value: expect.objectContaining({ + registries: [ + expect.objectContaining({ + id: 'scope:@corp-b', + }), + ], + }), + }), + { upsert: true, new: true } + ); expect(PluginSyncService.publish).toHaveBeenCalledWith('NPM_AUTH_UPDATED', {}); }); }); + + describe('syncNpmrc', () => { + it('should write .npmrc with multiple scoped registries and host auth lines', async () => { + const tokenA = encrypt('token_corp_a', mockEncryptionKey); + const tokenB = encrypt('token_corp_b', mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [ + { + id: 'scope:@corp-a', + scope: '@corp-a', + registry_url: 'https://npm.corp-a.com/', + encrypted_token: tokenA, + updated_at: new Date().toISOString(), + }, + { + id: 'scope:@corp-b', + scope: '@corp-b', + registry_url: 'https://npm.corp-b.com/', + encrypted_token: tokenB, + updated_at: new Date().toISOString(), + }, + ], + }, + }), + } as any); + + await NpmAuthService.syncNpmrc(); + + expect(fs.writeFileSync).toHaveBeenCalledWith( + NPMRC_PATH, + expect.stringContaining('@corp-a:registry=https://npm.corp-a.com/'), + expect.anything() + ); + expect(fs.writeFileSync).toHaveBeenCalledWith( + NPMRC_PATH, + expect.stringContaining('//npm.corp-a.com/:_authToken=token_corp_a'), + expect.anything() + ); + expect(fs.writeFileSync).toHaveBeenCalledWith( + NPMRC_PATH, + expect.stringContaining('@corp-b:registry=https://npm.corp-b.com/'), + expect.anything() + ); + expect(fs.writeFileSync).toHaveBeenCalledWith( + NPMRC_PATH, + expect.stringContaining('//npm.corp-b.com/:_authToken=token_corp_b'), + expect.anything() + ); + }); + }); }); From 5b47d43ebd495ef7619e7255e6eb3039b17f43e8 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 20 Sep 2026 02:30:10 +0530 Subject: [PATCH 21/46] feat: implement Kubernetes health probe server for background workers and processors --- src/processors/delayed/delayed.processor.ts | 33 +++++- src/processors/unified/unified.processor.ts | 30 +++++- src/types/types.ts | 22 ++++ src/utils/k8s-health-probe.ts | 113 ++++++++++++++++++++ src/workers/recovery/recovery.service.ts | 21 ++++ src/workers/worker.ts | 23 +++- 6 files changed, 238 insertions(+), 4 deletions(-) create mode 100644 src/utils/k8s-health-probe.ts diff --git a/src/processors/delayed/delayed.processor.ts b/src/processors/delayed/delayed.processor.ts index e065344..d6143a3 100644 --- a/src/processors/delayed/delayed.processor.ts +++ b/src/processors/delayed/delayed.processor.ts @@ -15,10 +15,13 @@ import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { connectRedis, disconnectRedis } from '@src/config/redis.config.js'; import { initTargetProducer, disconnectTargetProducer } from './target.producer.js'; import { initDLQStatusProducer, disconnectDLQStatusProducer } from './dlq.status.js'; -import { startDelayedConsumer, stopDelayedConsumer } from './delayed.consumer.js'; -import { startDelayedPoller, stopDelayedPoller } from './delayed.poller.js'; +import { startDelayedConsumer, stopDelayedConsumer, isConsumerActive } from './delayed.consumer.js'; +import { startDelayedPoller, stopDelayedPoller, isPollerActive } from './delayed.poller.js'; import { delayedWorkerLogger as logger } from '@src/workers/utils/logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; +import { getRedisClient } from '@src/config/redis.config.js'; +import { createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeServer } from '@src/types/types.js'; //Import the admin channel provider files here for them to self-register import "@src/admin-alerts/channels/discord.channel.js"; @@ -26,6 +29,7 @@ import "@src/admin-alerts/channels/telegram.channel.js"; let isShuttingDown = false; let dbConnection: Awaited> | null = null; +let probeServer: HealthProbeServer | null = null; /** * Graceful shutdown handler @@ -65,6 +69,11 @@ const gracefulShutdown = async (signal: string): Promise => { await dbConnection.disconnect(); } + // 6. Stop health probe server + if (probeServer) { + await probeServer.stop(); + } + logger.success('Graceful shutdown complete'); process.exit(0); } catch (err) { @@ -143,6 +152,26 @@ const main = async (): Promise => { logger.success('Delayed Processor is running!'); logger.info('================================'); + // 5. Start health probe server for Kubernetes + probeServer = createHealthProbeServer({ + serviceName: 'delayed-processor', + readinessChecks: [ + { name: 'redis', check: () => { + try { + return getRedisClient().status === 'ready'; + } catch { + return false; + } + }}, + { name: 'delayed_consumer', check: () => isConsumerActive() }, + { name: 'delayed_poller', check: () => isPollerActive() } + ], + livenessChecks: [ + { name: 'process', check: () => true } + ] + }); + await probeServer.start(); + // Register shutdown handlers registerShutdownHandlers(); } catch (err) { diff --git a/src/processors/unified/unified.processor.ts b/src/processors/unified/unified.processor.ts index 481a144..7016aff 100644 --- a/src/processors/unified/unified.processor.ts +++ b/src/processors/unified/unified.processor.ts @@ -29,6 +29,9 @@ import { } from './unified.consumer.js'; import { unifiedProcessorLogger as logger } from './unified.logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; +import { createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeServer } from '@src/types/types.js'; +import { getRedisClient } from '@src/config/redis.config.js'; // Import the admin channel provider files here for them to self-register import "@src/admin-alerts/channels/discord.channel.js"; @@ -38,6 +41,7 @@ import "@src/admin-alerts/channels/telegram.channel.js"; const activeChannels: string[] = []; let isShuttingDown = false; let dbConnection: { disconnect: () => Promise } | null = null; +let probeServer: HealthProbeServer | null = null; /** * Parse channel argument from CLI or environment @@ -133,6 +137,11 @@ const gracefulShutdown = async (signal: string): Promise => { await dbConnection.disconnect(); } + // 6. Stop health probe server + if (probeServer) { + await probeServer.stop(); + } + logger.success('Graceful shutdown complete'); process.exit(0); } catch (err) { @@ -269,7 +278,26 @@ const main = async (): Promise => { logger.success(`Unified Processor is running! Active channels: ${activeChannels.join(', ')}`); } - // 8. Register shutdown handlers + // 8. Start health probe server for Kubernetes + probeServer = createHealthProbeServer({ + serviceName: 'unified-processor', + readinessChecks: [ + { name: 'mongodb', check: () => dbConnection !== null }, + { name: 'redis', check: () => { + try { + return getRedisClient().status === 'ready'; + } catch { + return false; + } + }} + ], + livenessChecks: [ + { name: 'process', check: () => true } + ] + }); + await probeServer.start(); + + // 9. Register shutdown handlers registerShutdownHandlers(); logger.info('================================'); diff --git a/src/types/types.ts b/src/types/types.ts index f692be8..9c65fcc 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -495,3 +495,25 @@ export interface AdminAuthStatus { isConfigured: boolean; } +// ============================================================================ +// KUBERNETES HEALTH PROBE TYPES +// ============================================================================ + +export interface HealthProbeCheck { + name: string; + check: () => Promise | boolean; +} + +export interface HealthProbeOptions { + port?: number; + serviceName: string; + readinessChecks?: HealthProbeCheck[]; + livenessChecks?: HealthProbeCheck[]; +} + +export interface HealthProbeServer { + start: () => Promise; + stop: () => Promise; +} + + diff --git a/src/utils/k8s-health-probe.ts b/src/utils/k8s-health-probe.ts new file mode 100644 index 0000000..f289e83 --- /dev/null +++ b/src/utils/k8s-health-probe.ts @@ -0,0 +1,113 @@ +/** + * Lightweight Kubernetes Health Probe Server + * + * Provides HTTP endpoints (/healthz, /readyz) for background workers + * and processors to integrate cleanly with Kubernetes liveness and readiness probes. + * + * Uses Node's built-in http module for zero runtime overhead. + */ + +import http from 'http'; +import type { HealthProbeOptions, HealthProbeServer } from '@src/types/types.js'; + +export const createHealthProbeServer = (options: HealthProbeOptions): HealthProbeServer => { + const port = options.port ?? parseInt(process.env.PROBE_PORT || '8080', 10); + const serviceName = options.serviceName; + const readinessChecks = options.readinessChecks ?? []; + const livenessChecks = options.livenessChecks ?? []; + + let server: http.Server | null = null; + + const runChecks = async (checks: typeof readinessChecks): Promise<{ ok: boolean; results: Record }> => { + const results: Record = {}; + let allOk = true; + + for (const item of checks) { + try { + const passed = await item.check(); + results[item.name] = passed; + if (!passed) allOk = false; + } catch { + results[item.name] = false; + allOk = false; + } + } + + return { ok: allOk, results }; + }; + + const requestListener: http.RequestListener = async (req, res) => { + const url = req.url || ''; + + // Liveness probe + if (url === '/healthz' || url === '/livez') { + const { ok, results } = await runChecks(livenessChecks); + const status = ok ? 200 : 503; + res.writeHead(status, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + service: serviceName, + status: ok ? 'healthy' : 'unhealthy', + timestamp: new Date().toISOString(), + checks: results + })); + return; + } + + // Readiness probe + if (url === '/readyz' || url === '/ready') { + const { ok, results } = await runChecks(readinessChecks); + const status = ok ? 200 : 503; + res.writeHead(status, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + service: serviceName, + status: ok ? 'ready' : 'not_ready', + timestamp: new Date().toISOString(), + checks: results + })); + return; + } + + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Not Found' })); + }; + + return { + start: async (): Promise => { + // Allow disabling health probe explicitly via environment variable + if (process.env.ENABLE_HEALTH_PROBE === 'false') { + return; + } + + return new Promise((resolve, reject) => { + server = http.createServer(requestListener); + + server.on('error', (err: NodeJS.ErrnoException) => { + // If port is already in use (e.g. running multiple services locally in one container), + // log and do not crash the service + if (err.code === 'EADDRINUSE') { + console.warn(`[HealthProbe:${serviceName}] Port ${port} is already in use, health probe disabled.`); + resolve(); + return; + } + reject(err); + }); + + server.listen(port, () => { + console.info(`[HealthProbe:${serviceName}] Listening on port ${port} (/healthz, /readyz)`); + resolve(); + }); + }); + }, + + stop: async (): Promise => { + if (!server) return; + + return new Promise((resolve) => { + server!.close(() => { + server = null; + resolve(); + }); + }); + } + }; +}; diff --git a/src/workers/recovery/recovery.service.ts b/src/workers/recovery/recovery.service.ts index 1ff2643..4397a33 100644 --- a/src/workers/recovery/recovery.service.ts +++ b/src/workers/recovery/recovery.service.ts @@ -19,6 +19,8 @@ import { connectRedis, disconnectRedis, getRedisClient } from '@src/config/redis import { startRecoveryCron, stopRecoveryCron, setHealthChecker } from './recovery.cron.js'; import { recoveryLogger as logger, flushLogs } from '@src/workers/utils/logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; +import { createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeServer } from '@src/types/types.js'; //Import the admin channel provider files here for them to self-register import "@src/admin-alerts/channels/discord.channel.js"; @@ -27,6 +29,7 @@ import "@src/admin-alerts/channels/telegram.channel.js"; let isShuttingDown = false; let mongoReconnecting = false; let redisReconnecting = false; +let probeServer: HealthProbeServer | null = null; const RECONNECT_DELAY_MS = 5000; const MAX_RECONNECT_ATTEMPTS = 10; @@ -240,6 +243,11 @@ const shutdown = async (signal: string): Promise => { // Flush logs before exit await flushLogs(); + // Stop health probe server + if (probeServer) { + await probeServer.stop(); + } + logger.success('Recovery service shutdown complete'); process.exit(0); } catch (err) { @@ -280,6 +288,19 @@ const main = async (): Promise => { logger.info(`Processing stuck threshold: ${env.PROCESSING_STUCK_THRESHOLD_MS}ms`); logger.info(`Pending stuck threshold: ${env.PENDING_STUCK_THRESHOLD_MS}ms`); + // Start health probe server for Kubernetes + probeServer = createHealthProbeServer({ + serviceName: 'recovery-service', + readinessChecks: [ + { name: 'mongodb', check: () => isMongoHealthy() }, + { name: 'redis', check: () => isRedisHealthy() } + ], + livenessChecks: [ + { name: 'process', check: () => true } + ] + }); + await probeServer.start(); + // Register signal handlers for graceful shutdown process.on('SIGTERM', () => shutdown('SIGTERM')); process.on('SIGINT', () => shutdown('SIGINT')); diff --git a/src/workers/worker.ts b/src/workers/worker.ts index 1ccc129..d2f9247 100644 --- a/src/workers/worker.ts +++ b/src/workers/worker.ts @@ -2,9 +2,11 @@ import { connectMongoDB } from "@src/config/db.config.js"; import { dynamicConfig } from "@src/config/dynamic-config.service.js"; import { initProducer, disconnectProducer } from "@src/workers/producers/background.producer.js"; import { startCronJobs, stopCronJobs } from "@src/workers/cron/background.cron.js"; -import { startStatusConsumer, stopStatusConsumer } from "@src/workers/consumers/status.consumer.js"; +import { startStatusConsumer, stopStatusConsumer, isStatusConsumerRunning } from "@src/workers/consumers/status.consumer.js"; import { workerLogger as logger } from "@src/workers/utils/logger.js"; import { AdminAlertService } from "@src/admin-alerts/admin-alert.service.js"; +import { createHealthProbeServer } from "@src/utils/k8s-health-probe.js"; +import type { HealthProbeServer } from "@src/types/types.js"; //Import the admin channel provider files here for them to self-register import "@src/admin-alerts/channels/discord.channel.js"; @@ -12,6 +14,7 @@ import "@src/admin-alerts/channels/telegram.channel.js"; let isShuttingDown = false; let dbConnection: Awaited> | null = null; +let probeServer: HealthProbeServer | null = null; /** * Graceful shutdown handler @@ -44,6 +47,11 @@ const gracefulShutdown = async (signal: string): Promise => { await dbConnection.disconnect(); } + // 5. Stop health probe server + if (probeServer) { + await probeServer.stop(); + } + logger.success("Graceful shutdown complete"); process.exit(0); } catch (err) { @@ -117,6 +125,19 @@ const main = async (): Promise => { logger.success("Background Worker is running!"); logger.info("================================"); + // 5. Start health probe server for Kubernetes + probeServer = createHealthProbeServer({ + serviceName: 'worker', + readinessChecks: [ + { name: 'mongodb', check: () => dbConnection !== null }, + { name: 'status_consumer', check: () => isStatusConsumerRunning() } + ], + livenessChecks: [ + { name: 'process', check: () => true } + ] + }); + await probeServer.start(); + // Register shutdown handlers registerShutdownHandlers(); } catch (err) { From 0ae1c9ebb1ad27cde5771294b8f8d5db01c82a71 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 20 Sep 2026 19:02:06 +0530 Subject: [PATCH 22/46] feat(optimization): enhance outbox processing with 2-phase atomic claim and bulk insert for notifications --- src/api/utils/utils.ts | 19 +- src/config/db.config.ts | 7 +- src/database/models/notification.models.ts | 3 + src/database/models/status-outbox.models.ts | 1 + src/workers/cron/background.cron.ts | 218 ++++++++++++++---- tests/unit/workers/background.cron.test.ts | 243 ++++++++++++++++++++ 6 files changed, 436 insertions(+), 55 deletions(-) create mode 100644 tests/unit/workers/background.cron.test.ts diff --git a/src/api/utils/utils.ts b/src/api/utils/utils.ts index 47fd938..1905b71 100644 --- a/src/api/utils/utils.ts +++ b/src/api/utils/utils.ts @@ -406,6 +406,7 @@ export const process_notifications = async ( session.startTransaction(); const notification_ids: mongoose.Types.ObjectId[] = []; + const notifications_to_insert: InstanceType[] = []; const outbox_entries: outbox[] = []; const duplicate_keys: { request_id: string; channel: string }[] = []; @@ -415,7 +416,13 @@ export const process_notifications = async ( .findOne({ request_id: notification.request_id, channel: notification.channel, - status: { $ne: NOTIFICATION_STATUS.failed }, // Allow retrying failed notifications + status: { + $in: [ + NOTIFICATION_STATUS.pending, + NOTIFICATION_STATUS.processing, + NOTIFICATION_STATUS.delivered, + ], + }, }) .session(session); @@ -427,12 +434,11 @@ export const process_notifications = async ( continue; } - // Create notification document within transaction + // Create notification document within transaction (generates _id) const notification_doc = new notification_model(notification); - await notification_doc.save({ session }); - const notification_id = notification_doc._id as mongoose.Types.ObjectId; notification_ids.push(notification_id); + notifications_to_insert.push(notification_doc); // Create outbox entry const outbox_entry = convert_notification_schema_to_outbox_schema( @@ -456,6 +462,11 @@ export const process_notifications = async ( ); } + // Insert notifications in bulk within transaction + if (notifications_to_insert.length > 0) { + await notification_model.insertMany(notifications_to_insert, { session }); + } + // Insert outbox entries in bulk within transaction if (outbox_entries.length > 0) { await outbox_model.insertMany(outbox_entries, { session }); diff --git a/src/config/db.config.ts b/src/config/db.config.ts index 4d7856a..878cfaa 100644 --- a/src/config/db.config.ts +++ b/src/config/db.config.ts @@ -5,7 +5,12 @@ import { AdminAlertService } from "@src/admin-alerts/admin-alert.service.js"; export const connectMongoDB = async ()=>{ try{ - const db = await mongoose.connect(env.MONGO_URI); + const db = await mongoose.connect(env.MONGO_URI, { + maxPoolSize: 200, + minPoolSize: 20, + maxConnecting: 10, + serverSelectionTimeoutMS: 5000, + }); await db.connection.syncIndexes(); logger.info("MongoDB indexes synced"); diff --git a/src/database/models/notification.models.ts b/src/database/models/notification.models.ts index afbf7ce..be7b2a4 100644 --- a/src/database/models/notification.models.ts +++ b/src/database/models/notification.models.ts @@ -110,6 +110,9 @@ notification_schema.index( } ); +// Compound index for fast duplicate checking and status queries +notification_schema.index({ request_id: 1, channel: 1, status: 1 }); + const notification_model = mongoose.model('Notification', notification_schema); export default notification_model; \ No newline at end of file diff --git a/src/database/models/status-outbox.models.ts b/src/database/models/status-outbox.models.ts index 90ce3ac..38d361d 100644 --- a/src/database/models/status-outbox.models.ts +++ b/src/database/models/status-outbox.models.ts @@ -46,6 +46,7 @@ const status_outbox_schema = new mongoose.Schema( // Index for efficient polling of unprocessed entries status_outbox_schema.index({ processed: 1, created_at: 1 }); +status_outbox_schema.index({ processed: 1, claimed_by: 1, created_at: 1 }); // Index for claiming stale entries status_outbox_schema.index({ processed: 1, claimed_at: 1 }); diff --git a/src/workers/cron/background.cron.ts b/src/workers/cron/background.cron.ts index 4b27a34..2e10fa2 100644 --- a/src/workers/cron/background.cron.ts +++ b/src/workers/cron/background.cron.ts @@ -21,28 +21,32 @@ const state: OutboxCronState = { /** * Atomically claim a batch of pending outbox entries for this worker. - * Uses findOneAndUpdate in a loop to ensure no two workers claim the same event. - * Also reclaims stale entries from crashed workers. + * Uses a 2-phase batch claim to eliminate sequential round trips and lock contention: + * 1. Fast-path: query up to OUTBOX_BATCH_SIZE pending entries in FIFO order using index { status: 1, created_at: 1 } + * 2. Atomically update matching pending IDs to 'processing' with claimed_by and claimed_at + * 3. Fallback: if fewer than batch size claimed, reclaim stale entries from crashed workers */ const claimOutboxEntries = async (): Promise => { const claimedEntries: OutboxDocument[] = []; const now = new Date(); - const staleThreshold = new Date(now.getTime() - env.OUTBOX_CLAIM_TIMEOUT_MS); + const batchSize = env.OUTBOX_BATCH_SIZE; - // Claim entries one at a time atomically (or use bulkWrite for better performance) - for (let i = 0; i < env.OUTBOX_BATCH_SIZE; i++) { - // Try to claim a pending entry, or reclaim a stale processing entry - const entry = await outbox_model.findOneAndUpdate( + // 1. Fast-path: Find pending entries in FIFO order + const pendingCandidates = await outbox_model + .find({ status: OUTBOX_STATUS.pending }) + .sort({ created_at: 1 }) + .limit(batchSize) + .select('_id') + .lean(); + + if (pendingCandidates.length > 0) { + const candidateIds = pendingCandidates.map(c => c._id); + + // Atomically claim entries that are still pending + await outbox_model.updateMany( { - $or: [ - // Unclaimed pending entries - { status: OUTBOX_STATUS.pending }, - // Stale entries from crashed workers (claimed but not processed in time) - { - status: OUTBOX_STATUS.processing, - claimed_at: { $lt: staleThreshold } - } - ] + _id: { $in: candidateIds }, + status: OUTBOX_STATUS.pending }, { $set: { @@ -50,22 +54,69 @@ const claimOutboxEntries = async (): Promise => { claimed_by: env.WORKER_ID, claimed_at: now } - }, - { - returnDocument: 'after', - sort: { created_at: 1 } // FIFO order } ); - if (!entry) break; // No more entries to claim - claimedEntries.push(entry); + // Retrieve entries successfully claimed by this worker + const claimed = await outbox_model.find({ + _id: { $in: candidateIds }, + claimed_by: env.WORKER_ID, + claimed_at: now + }); + + claimedEntries.push(...claimed); + } + + // 2. Fallback: If quota remaining, reclaim stale entries from crashed workers + const remainingQuota = batchSize - claimedEntries.length; + if (remainingQuota > 0) { + const staleThreshold = new Date(now.getTime() - env.OUTBOX_CLAIM_TIMEOUT_MS); + + const staleCandidates = await outbox_model + .find({ + status: OUTBOX_STATUS.processing, + claimed_at: { $lt: staleThreshold } + }) + .sort({ created_at: 1 }) + .limit(remainingQuota) + .select('_id') + .lean(); + + if (staleCandidates.length > 0) { + const staleIds = staleCandidates.map(c => c._id); + + await outbox_model.updateMany( + { + _id: { $in: staleIds }, + status: OUTBOX_STATUS.processing, + claimed_at: { $lt: staleThreshold } + }, + { + $set: { + status: OUTBOX_STATUS.processing, + claimed_by: env.WORKER_ID, + claimed_at: now + } + } + ); + + const reclaimed = await outbox_model.find({ + _id: { $in: staleIds }, + claimed_by: env.WORKER_ID, + claimed_at: now + }); + + claimedEntries.push(...reclaimed); + } } return claimedEntries; }; /** - * Poll the outbox collection for pending events and send them to Kafka + * Poll the outbox collection for pending events and send them to Kafka. + * Continuously drains backlog while full batches are retrieved, preventing + * idle intervals during high ingestion load. */ const pollOutbox = async (): Promise => { if (state.isPolling || state.shouldStop) return; @@ -73,15 +124,22 @@ const pollOutbox = async (): Promise => { state.isPolling = true; try { - // Atomically claim entries for this worker - const claimedEntries = await claimOutboxEntries(); + while (!state.shouldStop) { + // Atomically claim entries for this worker + const claimedEntries = await claimOutboxEntries(); - if (claimedEntries.length === 0) return; + if (claimedEntries.length === 0) break; - logger.info(`Claimed ${claimedEntries.length} outbox entries (worker: ${env.WORKER_ID})`); + logger.info(`Claimed ${claimedEntries.length} outbox entries (worker: ${env.WORKER_ID})`); - const result = await sendOutboxEvents(claimedEntries); - logger.info(`Processed: ${result.successCount} success, ${result.failedCount} failed`); + const result = await sendOutboxEvents(claimedEntries); + logger.info(`Processed: ${result.successCount} success, ${result.failedCount} failed`); + + // If we claimed fewer than batch size, queue is drained for now + if (claimedEntries.length < env.OUTBOX_BATCH_SIZE) { + break; + } + } } catch (err) { logger.error("Error polling outbox:", err); } finally { @@ -126,41 +184,95 @@ const waitForOperationsToComplete = async (): Promise => { /** * Atomically claim a batch of pending status outbox entries for this worker. + * Uses 2-phase atomic batch claim to avoid sequential findOneAndUpdate round trips. */ const claimStatusOutboxEntries = async (): Promise => { const claimedEntries: status_outbox[] = []; const now = new Date(); - const staleThreshold = new Date(now.getTime() - env.OUTBOX_CLAIM_TIMEOUT_MS); + const batchSize = env.OUTBOX_BATCH_SIZE; + + // 1. Fast-path: Find unprocessed status outbox entries + const pendingCandidates = await status_outbox_model + .find({ processed: false, claimed_by: null }) + .sort({ created_at: 1 }) + .limit(batchSize) + .select('_id') + .lean(); + + if (pendingCandidates.length > 0) { + const candidateIds = pendingCandidates.map(c => c._id); - for (let i = 0; i < env.OUTBOX_BATCH_SIZE; i++) { - const entry = await status_outbox_model.findOneAndUpdate( + await status_outbox_model.updateMany( { - $or: [ - { processed: false, claimed_by: null }, - { processed: false, claimed_at: { $lt: staleThreshold } } - ] + _id: { $in: candidateIds }, + processed: false, + claimed_by: null }, { $set: { claimed_by: env.WORKER_ID, claimed_at: now } - }, - { - returnDocument: 'after', - sort: { created_at: 1 } } ); - if (!entry) break; - claimedEntries.push(entry); + const claimed = await status_outbox_model.find({ + _id: { $in: candidateIds }, + claimed_by: env.WORKER_ID, + claimed_at: now + }); + + claimedEntries.push(...claimed); + } + + // 2. Fallback: Check for stale processing entries + const remainingQuota = batchSize - claimedEntries.length; + if (remainingQuota > 0) { + const staleThreshold = new Date(now.getTime() - env.OUTBOX_CLAIM_TIMEOUT_MS); + + const staleCandidates = await status_outbox_model + .find({ + processed: false, + claimed_at: { $lt: staleThreshold } + }) + .sort({ created_at: 1 }) + .limit(remainingQuota) + .select('_id') + .lean(); + + if (staleCandidates.length > 0) { + const staleIds = staleCandidates.map(c => c._id); + + await status_outbox_model.updateMany( + { + _id: { $in: staleIds }, + processed: false, + claimed_at: { $lt: staleThreshold } + }, + { + $set: { + claimed_by: env.WORKER_ID, + claimed_at: now + } + } + ); + + const reclaimed = await status_outbox_model.find({ + _id: { $in: staleIds }, + claimed_by: env.WORKER_ID, + claimed_at: now + }); + + claimedEntries.push(...reclaimed); + } } return claimedEntries; }; /** - * Poll and process status outbox entries + * Poll and process status outbox entries. + * Continuously drains backlog while full batches are retrieved. */ const pollStatusOutbox = async (): Promise => { if (state.isPollingStatus || state.shouldStop) return; @@ -168,15 +280,21 @@ const pollStatusOutbox = async (): Promise => { state.isPollingStatus = true; try { - const claimedEntries = await claimStatusOutboxEntries(); - if (claimedEntries.length === 0) { - return; - } + while (!state.shouldStop) { + const claimedEntries = await claimStatusOutboxEntries(); + if (claimedEntries.length === 0) { + break; + } + + logger.info(`Claimed ${claimedEntries.length} status outbox entries (worker: ${env.WORKER_ID})`); - logger.info(`Claimed ${claimedEntries.length} status outbox entries (worker: ${env.WORKER_ID})`); + const result = await sendStatusOutboxEvents(claimedEntries); + logger.info(`Status outbox: ${result.successCount} success, ${result.failedCount} failed`); - const result = await sendStatusOutboxEvents(claimedEntries); - logger.info(`Status outbox: ${result.successCount} success, ${result.failedCount} failed`); + if (claimedEntries.length < env.OUTBOX_BATCH_SIZE) { + break; + } + } } catch (err) { logger.error("Error polling status outbox:", err); } finally { diff --git a/tests/unit/workers/background.cron.test.ts b/tests/unit/workers/background.cron.test.ts new file mode 100644 index 0000000..5389394 --- /dev/null +++ b/tests/unit/workers/background.cron.test.ts @@ -0,0 +1,243 @@ +/** + * Unit Tests for Background Outbox Cron Jobs + * Tests the batch 2-phase atomic claim, eager draining loop, and cleanup routines. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import mongoose from 'mongoose'; +import { OUTBOX_STATUS } from '../../../src/types/types.js'; + +// Mock dependencies +const mockOutboxModel = { + find: vi.fn(), + updateMany: vi.fn(), + deleteMany: vi.fn(), +}; + +const mockStatusOutboxModel = { + find: vi.fn(), + updateMany: vi.fn(), +}; + +const mockBackgroundProducer = { + sendOutboxEvents: vi.fn(), + sendStatusOutboxEvents: vi.fn(), +}; + +const mockLogger = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + success: vi.fn(), +}; + +vi.mock('@src/database/models/outbox.models.js', () => ({ + default: mockOutboxModel, +})); + +vi.mock('@src/database/models/status-outbox.models.js', () => ({ + default: mockStatusOutboxModel, +})); + +vi.mock('@src/workers/producers/background.producer.js', () => ({ + sendOutboxEvents: (...args: unknown[]) => mockBackgroundProducer.sendOutboxEvents(...args), + sendStatusOutboxEvents: (...args: unknown[]) => mockBackgroundProducer.sendStatusOutboxEvents(...args), +})); + +vi.mock('@src/workers/utils/logger.js', () => ({ + cronLogger: mockLogger, +})); + +vi.mock('@src/config/env.config.js', () => ({ + env: { + WORKER_ID: 'test-worker-1', + OUTBOX_POLL_INTERVAL_MS: 5000, + OUTBOX_CLEANUP_INTERVAL_MS: 60000, + OUTBOX_BATCH_SIZE: 10, + OUTBOX_RETENTION_MS: 300000, + OUTBOX_CLAIM_TIMEOUT_MS: 30000, + }, +})); + +vi.mock('@src/config/dynamic-config.service.js', () => ({ + dynamicConfig: { + on: vi.fn(), + }, +})); + +describe('Background Cron Unit Tests', () => { + let cronModule: typeof import('../../../src/workers/cron/background.cron.js'); + + const createEmptyChain = () => ({ + sort: vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ + select: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue([]), + }), + }), + }), + }); + + beforeEach(async () => { + vi.clearAllMocks(); + mockOutboxModel.find.mockImplementation(() => createEmptyChain()); + mockStatusOutboxModel.find.mockImplementation(() => createEmptyChain()); + cronModule = await import('../../../src/workers/cron/background.cron.js'); + }); + + afterEach(async () => { + await cronModule.stopCronJobs(); + }); + + it('should manage cron job lifecycle (start, status, stop)', async () => { + expect(cronModule.isCronRunning()).toBe(false); + + cronModule.startCronJobs(); + expect(cronModule.isCronRunning()).toBe(true); + + // Calling start again should be a no-op + cronModule.startCronJobs(); + expect(cronModule.isCronRunning()).toBe(true); + + await cronModule.stopCronJobs(); + expect(cronModule.isCronRunning()).toBe(false); + }); + + it('should claim pending outbox entries using 2-phase atomic batch claim and drain eagerly', async () => { + const id1 = new mongoose.Types.ObjectId(); + const id2 = new mongoose.Types.ObjectId(); + + // Round 1: find returns 2 candidates (less than batch size 10), claiming finishes + const mockCandidates = [{ _id: id1 }, { _id: id2 }]; + const mockClaimedDocs = [ + { _id: id1, status: OUTBOX_STATUS.processing, topic: 'notification_mock' }, + { _id: id2, status: OUTBOX_STATUS.processing, topic: 'notification_mock' }, + ]; + + // Chain for outbox_model.find().sort().limit().select().lean() + const selectLeanMock = { + select: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue(mockCandidates), + }), + }; + const limitMock = { + limit: vi.fn().mockReturnValue(selectLeanMock), + }; + const sortMock = { + sort: vi.fn().mockReturnValue(limitMock), + }; + + // When finding candidates vs finding claimed entries: + mockOutboxModel.find.mockImplementation((query: Record) => { + if (query.status === OUTBOX_STATUS.pending) { + return sortMock; + } + if (query.status === OUTBOX_STATUS.processing && query.claimed_at) { + // Stale check: returns empty + return { + sort: vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ + select: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue([]), + }), + }), + }), + }; + } + // Finding claimed docs by _id: + return Promise.resolve(mockClaimedDocs); + }); + + mockOutboxModel.updateMany.mockResolvedValue({ modifiedCount: 2 }); + mockBackgroundProducer.sendOutboxEvents.mockResolvedValue({ successCount: 2, failedCount: 0 }); + + // Trigger cron jobs (which immediately runs pollOutbox) + cronModule.startCronJobs(); + await new Promise(resolve => setTimeout(resolve, 50)); + + // Verify updateMany was called atomically + expect(mockOutboxModel.updateMany).toHaveBeenCalledWith( + { _id: { $in: [id1, id2] }, status: OUTBOX_STATUS.pending }, + expect.objectContaining({ + $set: expect.objectContaining({ + status: OUTBOX_STATUS.processing, + claimed_by: 'test-worker-1', + }), + }) + ); + + // Verify sendOutboxEvents was called with the claimed docs + expect(mockBackgroundProducer.sendOutboxEvents).toHaveBeenCalledWith(mockClaimedDocs); + }); + + it('should reclaim stale entries when pending quota is remaining', async () => { + const staleId = new mongoose.Types.ObjectId(); + const staleDoc = { _id: staleId, status: OUTBOX_STATUS.processing, topic: 'notification_mock' }; + + // No pending candidates + const emptyPending = { + sort: vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ + select: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue([]), + }), + }), + }), + }; + + // Stale candidates found + const staleCandidates = { + sort: vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ + select: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: staleId }]), + }), + }), + }), + }; + + mockOutboxModel.find.mockImplementation((query: Record) => { + if (query.status === OUTBOX_STATUS.pending) { + return emptyPending; + } + if (query.status === OUTBOX_STATUS.processing && query.claimed_at) { + return staleCandidates; + } + return Promise.resolve([staleDoc]); + }); + + mockOutboxModel.updateMany.mockResolvedValue({ modifiedCount: 1 }); + mockBackgroundProducer.sendOutboxEvents.mockResolvedValue({ successCount: 1, failedCount: 0 }); + + cronModule.startCronJobs(); + await new Promise(resolve => setTimeout(resolve, 50)); + + expect(mockOutboxModel.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + _id: { $in: [staleId] }, + status: OUTBOX_STATUS.processing, + }), + expect.any(Object) + ); + expect(mockBackgroundProducer.sendOutboxEvents).toHaveBeenCalledWith([staleDoc]); + }); + + it('should cleanup published events older than retention period', async () => { + vi.useFakeTimers(); + try { + mockOutboxModel.deleteMany.mockResolvedValue({ deletedCount: 5 }); + + cronModule.startCronJobs(); + // Advance time to trigger cleanup interval (60,000ms) + await vi.advanceTimersByTimeAsync(60000); + + expect(mockOutboxModel.deleteMany).toHaveBeenCalledWith( + expect.objectContaining({ + status: OUTBOX_STATUS.published, + }) + ); + } finally { + vi.useRealTimers(); + } + }); +}); From eba09dcf27752d7123a497765e53dc9b119e35fa Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 20 Sep 2026 19:33:56 +0530 Subject: [PATCH 23/46] feat(health): implement consumer health monitoring and reporting --- src/processors/unified/unified.consumer.ts | 178 +++++++++++++++++- src/processors/unified/unified.processor.ts | 86 ++++++--- src/types/types.ts | 22 +++ .../unit/processors/unified.consumer.test.ts | 71 +++++++ 4 files changed, 325 insertions(+), 32 deletions(-) diff --git a/src/processors/unified/unified.consumer.ts b/src/processors/unified/unified.consumer.ts index 7fd5187..a285f4c 100644 --- a/src/processors/unified/unified.consumer.ts +++ b/src/processors/unified/unified.consumer.ts @@ -10,7 +10,7 @@ * - Retry queue management */ -import { Consumer, EachMessagePayload } from 'kafkajs'; +import { Consumer, EachMessagePayload, ConsumerCrashEvent } from 'kafkajs'; import { kafka, ensureChannelTopic } from '@src/config/kafka.config.js'; import { NOTIFICATION_STATUS_SF } from '@src/types/types.js'; import { env } from '@src/config/env.config.js'; @@ -23,7 +23,13 @@ import { resolveFallbackProviderId, validateNotificationForProvider } from '@src/plugins/index.js'; -import type { BaseNotification, DeliveryResult } from '@src/types/types.js'; +import type { + BaseNotification, + DeliveryResult, + ConsumerHealthState, + ConsumerHealthCheckResult, + ConsumerHealthDetails +} from '@src/types/types.js'; // Shared utilities import { tryAcquireProcessingLock, setDelivered, setFailed, setRateLimited } from '@src/processors/shared/idempotency.js'; @@ -33,9 +39,14 @@ import { publishDelayed, buildDelayedPayloadGeneric } from '@src/processors/shar import { handleSchemaValidationFailure } from '../shared/schema-failure-handler.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; import status_outbox_model from '@src/database/models/status-outbox.models.js'; -// Track active consumers by channel + +// Re-export consumer health types +export type { ConsumerHealthState, ConsumerHealthCheckResult, ConsumerHealthDetails }; + +// Track active consumers and their health status by channel const consumers: Map = new Map(); const consumingState: Map = new Map(); +const consumerHealthState: Map = new Map(); /** * Get Kafka topic name for a channel @@ -462,6 +473,71 @@ export const startUnifiedConsumer = async (channel: string): Promise => { rebalanceTimeout: 60000, heartbeatInterval: 3000, }); + + const health: ConsumerHealthState = { + channel, + isRunning: false, + hasCrashed: false, + lastHeartbeat: Date.now(), + }; + consumerHealthState.set(channel, health); + + // Register Kafka lifecycle and crash event listeners + if (typeof consumer.on === 'function' && consumer.events) { + consumer.on(consumer.events.HEARTBEAT, () => { + const h = consumerHealthState.get(channel); + if (h) { + h.lastHeartbeat = Date.now(); + } + }); + + consumer.on(consumer.events.CRASH, async (event: ConsumerCrashEvent) => { + const h = consumerHealthState.get(channel); + const crashError = event?.payload?.error; + const errorMsg = crashError ? String(crashError.message || crashError) : 'Unknown crash'; + if (h) { + h.hasCrashed = true; + h.isRunning = false; + h.crashReason = errorMsg; + } + logger.error(`[${channel}] Consumer CRASHED: ${errorMsg}`, crashError); + + void AdminAlertService.sendAlert( + 'service_health', + `🔴 KAFKA CONSUMER CRASHED - Unified Processor\n` + + `Channel: ${channel}\n` + + `Group: ${groupId}\n` + + `Restartable: ${Boolean(event?.payload?.restart)}\n` + + `Error: ${errorMsg}\n` + + `Action: Kubernetes liveness probe will detect failure and restart container.`, + { severity: 'critical' } + ); + }); + + consumer.on(consumer.events.STOP, () => { + const h = consumerHealthState.get(channel); + if (h) { + h.isRunning = false; + } + logger.warn(`[${channel}] Consumer stopped event received`); + }); + + consumer.on(consumer.events.DISCONNECT, () => { + const h = consumerHealthState.get(channel); + if (h) { + h.isRunning = false; + } + logger.warn(`[${channel}] Consumer disconnected`); + }); + + consumer.on(consumer.events.CONNECT, () => { + const h = consumerHealthState.get(channel); + if (h) { + h.isRunning = true; + h.lastHeartbeat = Date.now(); + } + }); + } await consumer.connect(); await consumer.subscribe({ topic, fromBeginning: false }); @@ -476,6 +552,11 @@ export const startUnifiedConsumer = async (channel: string): Promise => { if (!consumingState.get(channel)) { return; // Consumer is stopping } + + const h = consumerHealthState.get(channel); + if (h) { + h.lastHeartbeat = Date.now(); + } const shouldCommit = await processMessage(channel, payload); @@ -494,6 +575,9 @@ export const startUnifiedConsumer = async (channel: string): Promise => { } }); + health.isRunning = true; + health.lastHeartbeat = Date.now(); + logger.success(`[${channel}] Consumer started`); }; @@ -508,12 +592,17 @@ export const stopUnifiedConsumer = async (channel: string): Promise => { logger.info(`[${channel}] Stopping consumer...`); consumingState.set(channel, false); + const health = consumerHealthState.get(channel); + if (health) { + health.isRunning = false; + } try { await consumer.stop(); await consumer.disconnect(); consumers.delete(channel); consumingState.delete(channel); + consumerHealthState.delete(channel); logger.info(`[${channel}] Consumer stopped`); } catch (err) { logger.error(`[${channel}] Error stopping consumer:`, err); @@ -536,3 +625,86 @@ export const stopAllConsumers = async (): Promise => { export const getActiveConsumerChannels = (): string[] => { return Array.from(consumers.keys()); }; + +/** + * Check whether consumers for specified channels (or all active consumers) are healthy. + * + * @param expectedChannels Optional list of channels expected to be running. + * @param maxHeartbeatStalenessMs Max ms since last heartbeat before marking stalled (default 90s). + */ +export const areUnifiedConsumersHealthy = ( + expectedChannels?: string[], + maxHeartbeatStalenessMs = 90000 +): ConsumerHealthCheckResult => { + const details: Record = {}; + const now = Date.now(); + let allHealthy = true; + + const channelsToCheck = expectedChannels && expectedChannels.length > 0 + ? expectedChannels + : Array.from(consumers.keys()); + + // In standby mode (no channels expected/running), considered healthy + if (channelsToCheck.length === 0) { + return { healthy: true, details: {} }; + } + + for (const channel of channelsToCheck) { + const health = consumerHealthState.get(channel); + const consumer = consumers.get(channel); + + if (!health || !consumer) { + details[channel] = { + running: false, + crashed: false, + error: 'Consumer not initialized or missing', + secondsSinceHeartbeat: -1 + }; + allHealthy = false; + continue; + } + + const secondsSinceHeartbeat = Math.round((now - health.lastHeartbeat) / 1000); + + if (health.hasCrashed) { + details[channel] = { + running: false, + crashed: true, + error: health.crashReason || 'Consumer crashed', + secondsSinceHeartbeat + }; + allHealthy = false; + continue; + } + + if (!health.isRunning) { + details[channel] = { + running: false, + crashed: false, + error: 'Consumer is stopped or disconnected', + secondsSinceHeartbeat + }; + allHealthy = false; + continue; + } + + if (now - health.lastHeartbeat > maxHeartbeatStalenessMs) { + details[channel] = { + running: true, + crashed: false, + error: `Heartbeat stalled (${secondsSinceHeartbeat}s ago)`, + secondsSinceHeartbeat + }; + allHealthy = false; + continue; + } + + details[channel] = { + running: true, + crashed: false, + secondsSinceHeartbeat + }; + } + + return { healthy: allHealthy, details }; +}; diff --git a/src/processors/unified/unified.processor.ts b/src/processors/unified/unified.processor.ts index 7016aff..37c8c30 100644 --- a/src/processors/unified/unified.processor.ts +++ b/src/processors/unified/unified.processor.ts @@ -26,6 +26,7 @@ import { stopUnifiedConsumer, stopAllConsumers, getActiveConsumerChannels, + areUnifiedConsumersHealthy, } from './unified.consumer.js'; import { unifiedProcessorLogger as logger } from './unified.logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; @@ -61,21 +62,31 @@ const getChannelConfig = (): string[] | 'all' => { return 'all'; }; +/** + * Compute the list of channels this processor is currently configured to handle + */ +const getExpectedChannels = (): string[] => { + const loadedChannels = PluginRegistry.getChannels(); + const channelConfig = getChannelConfig(); + + if (channelConfig === 'all') { + return loadedChannels; + } + + return channelConfig.filter((ch) => loadedChannels.includes(ch)); +}; + /** * Check for new channels and start consumers if configured */ const syncActiveConsumers = async (): Promise => { if (isShuttingDown) return; - const loadedChannels = PluginRegistry.getChannels(); const currentRunning = getActiveConsumerChannels(); - const channelConfig = getChannelConfig(); - - for (const channel of loadedChannels) { - const shouldRun = - channelConfig === 'all' || (Array.isArray(channelConfig) && channelConfig.includes(channel)); + const expectedChannels = getExpectedChannels(); - if (shouldRun && !currentRunning.includes(channel)) { + for (const channel of expectedChannels) { + if (!currentRunning.includes(channel)) { try { logger.info(`Starting dynamic consumer for new channel: ${channel}`); await startUnifiedConsumer(channel); @@ -88,9 +99,9 @@ const syncActiveConsumers = async (): Promise => { } } - // If a channel was removed from PluginRegistry, stop its consumer + // If a channel was removed or is no longer expected, stop its consumer for (const runningChannel of currentRunning) { - if (!loadedChannels.includes(runningChannel)) { + if (!expectedChannels.includes(runningChannel)) { logger.info(`Stopping consumer for removed channel: ${runningChannel}`); await stopUnifiedConsumer(runningChannel); const index = activeChannels.indexOf(runningChannel); @@ -249,20 +260,10 @@ const main = async (): Promise => { await initDelayedProducer(); // 6. Determine which channels to process - const channelConfig = getChannelConfig(); - let channelsToProcess: string[] = []; - - if (channelConfig === 'all') { - channelsToProcess = loadedChannels; + const channelsToProcess = getExpectedChannels(); + if (getChannelConfig() === 'all') { logger.info('Configured for multi-channel mode (all channels)'); } else { - channelsToProcess = channelConfig.filter((ch) => { - if (!loadedChannels.includes(ch)) { - logger.warn(`Channel '${ch}' not found in loaded plugins, skipping`); - return false; - } - return true; - }); logger.info(`Configured for single-channel mode: ${channelsToProcess.join(', ')}`); } @@ -282,17 +283,44 @@ const main = async (): Promise => { probeServer = createHealthProbeServer({ serviceName: 'unified-processor', readinessChecks: [ - { name: 'mongodb', check: () => dbConnection !== null }, - { name: 'redis', check: () => { - try { - return getRedisClient().status === 'ready'; - } catch { - return false; + { name: 'mongodb', check: () => !isShuttingDown && dbConnection !== null }, + { + name: 'redis', + check: () => { + if (isShuttingDown) return false; + try { + return getRedisClient().status === 'ready'; + } catch { + return false; + } } - }} + }, + { + name: 'kafka_consumers', + check: () => { + if (isShuttingDown) return false; + const expected = getExpectedChannels(); + const result = areUnifiedConsumersHealthy(expected); + return result.healthy; + } + } ], livenessChecks: [ - { name: 'process', check: () => true } + { + name: 'process', + check: () => !isShuttingDown + }, + { + name: 'kafka_consumers', + check: () => { + if (isShuttingDown) return false; + const expected = getExpectedChannels(); + // Standby mode (no channels configured yet) is considered healthy + if (expected.length === 0) return true; + const result = areUnifiedConsumersHealthy(expected); + return result.healthy; + } + } ] }); await probeServer.start(); diff --git a/src/types/types.ts b/src/types/types.ts index 9c65fcc..ed75ddd 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -516,4 +516,26 @@ export interface HealthProbeServer { stop: () => Promise; } +// ============================================================================ +// PROCESSOR CONSUMER HEALTH TYPES +// ============================================================================ + +export interface ConsumerHealthState { + channel: string; + isRunning: boolean; + hasCrashed: boolean; + crashReason?: string; + lastHeartbeat: number; +} +export interface ConsumerHealthDetails { + running: boolean; + crashed: boolean; + error?: string; + secondsSinceHeartbeat: number; +} + +export interface ConsumerHealthCheckResult { + healthy: boolean; + details: Record; +} diff --git a/tests/unit/processors/unified.consumer.test.ts b/tests/unit/processors/unified.consumer.test.ts index 571dc42..42fa451 100644 --- a/tests/unit/processors/unified.consumer.test.ts +++ b/tests/unit/processors/unified.consumer.test.ts @@ -36,6 +36,14 @@ const kafkaConsumerMock = { commitOffsets: commitOffsetsMock, stop: vi.fn().mockResolvedValue(undefined), disconnect: vi.fn().mockResolvedValue(undefined), + on: vi.fn(), + events: { + CRASH: 'consumer.crash', + HEARTBEAT: 'consumer.heartbeat', + STOP: 'consumer.stop', + DISCONNECT: 'consumer.disconnect', + CONNECT: 'consumer.connect', + }, }; vi.mock('../../../src/config/kafka.config.js', () => ({ @@ -348,3 +356,66 @@ describe('processMessage via consumer run loop', () => { expect(commitOffsetsMock).toHaveBeenCalledTimes(1); }); }); + +describe('areUnifiedConsumersHealthy', () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + }); + + it('should report healthy when in standby mode with no expected channels', async () => { + const { areUnifiedConsumersHealthy } = await import('../../../src/processors/unified/unified.consumer.js'); + const result = areUnifiedConsumersHealthy([]); + expect(result.healthy).toBe(true); + expect(result.details).toEqual({}); + }); + + it('should report unhealthy if expected channel has no consumer running', async () => { + const { areUnifiedConsumersHealthy } = await import('../../../src/processors/unified/unified.consumer.js'); + const result = areUnifiedConsumersHealthy(['email']); + expect(result.healthy).toBe(false); + expect(result.details.email.running).toBe(false); + expect(result.details.email.error).toContain('not initialized or missing'); + }); + + it('should report healthy once consumer starts and receives heartbeat', async () => { + const { startUnifiedConsumer, areUnifiedConsumersHealthy, stopAllConsumers } = await import('../../../src/processors/unified/unified.consumer.js'); + await startUnifiedConsumer('email'); + + const result = areUnifiedConsumersHealthy(['email']); + expect(result.healthy).toBe(true); + expect(result.details.email.running).toBe(true); + expect(result.details.email.crashed).toBe(false); + + await stopAllConsumers(); + }); + + it('should report unhealthy if consumer has crashed', async () => { + let crashListener: ((event: unknown) => void) | undefined; + kafkaConsumerMock.on.mockImplementation((event: string, listener: (e: unknown) => void) => { + if (event === 'consumer.crash') { + crashListener = listener; + } + }); + + const { startUnifiedConsumer, areUnifiedConsumersHealthy, stopAllConsumers } = await import('../../../src/processors/unified/unified.consumer.js'); + await startUnifiedConsumer('email'); + + // Simulate Kafka consumer crash event + if (crashListener) { + crashListener({ + payload: { + error: new Error('Kafka broker connection lost'), + restart: false, + } + }); + } + + const result = areUnifiedConsumersHealthy(['email']); + expect(result.healthy).toBe(false); + expect(result.details.email.crashed).toBe(true); + expect(result.details.email.error).toContain('Kafka broker connection lost'); + + await stopAllConsumers(); + }); +}); From eff202be248523e452593d3fd1c13aa1ba5126ae Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Mon, 21 Sep 2026 07:49:20 +0530 Subject: [PATCH 24/46] feat: add rate limit retry delay configuration and implement cascading provider fallback - Introduced `rate_limit_retry_delay_ms` to operational settings for dynamic configuration. - Updated dashboard settings to allow administrators to configure rate limit retry delay. - Enhanced rate limiter to use fixed retry delay instead of staggered delay based on queue position. - Implemented cascading provider fallback mechanism in the unified consumer to handle rate-limited providers. - Added utility function for exponential backoff calculation. - Updated tests to cover new functionality and ensure proper behavior of rate limiting and fallback logic. --- AGENTS.md | 6 ++ dashboard/app/settings/page.tsx | 21 +++++ dashboard/lib/types.ts | 1 + src/config/dynamic-config.service.ts | 1 + src/config/env.config.ts | 3 + src/plugins/index.ts | 1 + src/plugins/loader/router.ts | 41 +++++++- src/processors/delayed/delayed.poller.ts | 5 +- src/processors/shared/delayed.producer.ts | 6 +- src/processors/shared/rate-limiter.ts | 41 +++----- src/processors/unified/unified.consumer.ts | 94 +++++++++++++------ src/types/schemas.ts | 1 + src/utils/backoff.utils.ts | 22 +++++ tests/unit/config/dynamic-config.test.ts | 7 ++ tests/unit/plugins/router.test.ts | 35 +++++++ .../unit/processors/unified.consumer.test.ts | 72 ++++++++++---- tests/unit/shared/rate-limiter.test.ts | 67 ++++++------- tests/unit/utils/backoff.utils.test.ts | 29 ++++++ 18 files changed, 333 insertions(+), 120 deletions(-) create mode 100644 src/utils/backoff.utils.ts create mode 100644 tests/unit/utils/backoff.utils.test.ts diff --git a/AGENTS.md b/AGENTS.md index d943481..d912bc4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,5 +34,11 @@ Use short Conventional Commit style messages such as `feat: add email retry poli PRs should include a summary, linked issue when applicable, test evidence, and screenshots or GIFs for `dashboard/` UI changes. Husky `pre-push` runs both lint commands. +## Dynamic Configuration & Dashboard Settings +Whenever any configuration field is added, modified, or removed in the dynamic operational configuration (`src/types/schemas.ts` in `operationalSettingsSchema`, `src/config/env.config.ts`, or `src/config/dynamic-config.service.ts`), you **must** also update the dashboard: +1. Update the `OperationalSettings` interface in `dashboard/lib/types.ts`. +2. Update the corresponding card and input fields in `dashboard/app/settings/page.tsx` so administrators can view and tune the parameter live with appropriate labels, inputs, and validation hints. + ## Security & Configuration Tips Do not commit secrets. Use `.env.example` as the template for `.env`, and update it when adding configuration. For local full-stack work, use `./docker-compose/scripts/switch-env.ps1 -Env local -Rebuild` (or `docker compose -f docker-compose/docker-compose.local.yaml up -d`); for shared infrastructure, use `docker compose -f docker-compose/docker-compose.infra.yaml up -d` (or `./docker-compose/scripts/switch-env.ps1 -Env infra-only`). + diff --git a/dashboard/app/settings/page.tsx b/dashboard/app/settings/page.tsx index 3f54ed0..704b928 100644 --- a/dashboard/app/settings/page.tsx +++ b/dashboard/app/settings/page.tsx @@ -498,6 +498,27 @@ export default function SettingsPage() { Active processing mutex duration (default: 120s)
+ +
+ + + updateGroupField( + "retry", + "rate_limit_retry_delay_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Delay when all cascade providers are rate-limited (Safe: 100 - 60,000 ms) + +
diff --git a/dashboard/lib/types.ts b/dashboard/lib/types.ts index d0605d1..2204fe0 100644 --- a/dashboard/lib/types.ts +++ b/dashboard/lib/types.ts @@ -323,6 +323,7 @@ export interface OperationalSettings { max_retry_count: number; idempotency_ttl_seconds: number; processing_ttl_seconds: number; + rate_limit_retry_delay_ms: number; }; delayed: { delayed_poll_interval_ms: number; diff --git a/src/config/dynamic-config.service.ts b/src/config/dynamic-config.service.ts index 592e18c..8205874 100644 --- a/src/config/dynamic-config.service.ts +++ b/src/config/dynamic-config.service.ts @@ -35,6 +35,7 @@ export const SYSTEM_DEFAULT_OPERATIONAL_SETTINGS: OperationalSettings = { max_retry_count: 5, idempotency_ttl_seconds: 86400, processing_ttl_seconds: 120, + rate_limit_retry_delay_ms: 5000, }, delayed: { delayed_poll_interval_ms: 1000, diff --git a/src/config/env.config.ts b/src/config/env.config.ts index 3d1062c..4063d13 100644 --- a/src/config/env.config.ts +++ b/src/config/env.config.ts @@ -76,6 +76,9 @@ export const env = { get PROCESSING_TTL_SECONDS(): number { return dynamicConfig?.get?.('retry')?.processing_ttl_seconds ?? 120; }, + get RATE_LIMIT_RETRY_DELAY_MS(): number { + return dynamicConfig?.get?.('retry')?.rate_limit_retry_delay_ms ?? 5000; + }, // Delayed Worker get MAX_POLLER_RETRIES(): number { diff --git a/src/plugins/index.ts b/src/plugins/index.ts index 264a63e..85be2a0 100644 --- a/src/plugins/index.ts +++ b/src/plugins/index.ts @@ -30,6 +30,7 @@ export { sendToProvider, validateNotification, validateNotificationForProvider, + getProviderCascade, resolveFallbackProviderId, getRateLimitConfig } from './loader/router.js'; diff --git a/src/plugins/loader/router.ts b/src/plugins/loader/router.ts index cecb6d6..8f5cb1b 100644 --- a/src/plugins/loader/router.ts +++ b/src/plugins/loader/router.ts @@ -9,6 +9,7 @@ import type { DeliveryResult, BaseNotification } from '@src/types/types.js'; import { unifiedProcessorLogger as logger } from '@src/processors/unified/unified.logger.js'; import { handleSchemaValidationFailure } from '@src/processors/shared/schema-failure-handler.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; +import { consumeToken } from '@src/processors/shared/rate-limiter.js'; /** * Try fallback providers when primary fails with non-retryable error. @@ -26,9 +27,10 @@ async function tryFallback( return null; } - const candidateIds = fallbackIds.filter(id => id !== currentProviderId); + const currentIndex = currentProviderId ? fallbackIds.indexOf(currentProviderId) : -1; + const candidateIds = currentIndex >= 0 ? fallbackIds.slice(currentIndex + 1) : fallbackIds.filter(id => id !== currentProviderId); if (candidateIds.length === 0) { - logger.debug(`[ProviderRouter] Fallback providers match current provider (${currentProviderId}), skipping fallback`); + logger.debug(`[ProviderRouter] No further fallback providers in cascade after ${currentProviderId}`); return null; } @@ -42,6 +44,18 @@ async function tryFallback( continue; } + // Check rate limiting before attempting to send via fallback + const rateLimitResult = await consumeToken(fallbackId); + if (!rateLimitResult.allowed) { + logger.warn(`[ProviderRouter] Fallback provider '${fallbackId}' is rate limited, continuing cascade`); + lastError = { + code: 'FALLBACK_RATE_LIMITED', + message: `Fallback provider '${fallbackId}' is rate limited`, + retryable: true + }; + continue; + } + logger.info(`[ProviderRouter] Trying fallback provider [${i + 1}/${candidateIds.length}]: ${fallbackId} (${fallbackProvider.manifest.displayName})`); void AdminAlertService.sendAlert('service_health', @@ -120,6 +134,29 @@ async function tryFallback( }; } +/** + * Get ordered provider cascade for a channel starting from currentProviderId (or default provider). + */ +export function getProviderCascade( + channel: string, + currentProviderId?: string +): string[] { + const defaultId = PluginRegistry.getDefaultProviderId(channel); + const fallbackIds = PluginRegistry.getFallbackProviderIds(channel) || []; + const allProviders = defaultId ? [defaultId, ...fallbackIds] : [...fallbackIds]; + + if (!currentProviderId) { + return allProviders; + } + + const currentIndex = allProviders.indexOf(currentProviderId); + if (currentIndex >= 0) { + return allProviders.slice(currentIndex); + } + + return allProviders; +} + /** * Resolve a fallback provider ID for retry exhaustion handoff. * Returns undefined when no distinct fallback provider is available. diff --git a/src/processors/delayed/delayed.poller.ts b/src/processors/delayed/delayed.poller.ts index f205111..e975a4d 100644 --- a/src/processors/delayed/delayed.poller.ts +++ b/src/processors/delayed/delayed.poller.ts @@ -16,6 +16,7 @@ import { publishDLQFailureStatus } from './dlq.status.js'; import { delayedWorkerLogger as logger } from '@src/workers/utils/logger.js'; import type { DelayedEventWithRetries } from '@src/types/types.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; +import { calculateExponentialBackoff } from '@src/utils/backoff.utils.js'; let pollerInterval: NodeJS.Timeout | null = null; let isPolling = false; @@ -29,9 +30,7 @@ let isPolling = false; * @returns Delay in milliseconds (5s, 10s, 20s, 40s, capped at 60s) */ const calculateBackoff = (retryCount: number): number => { - const baseDelay = 5000; // 5 seconds - const maxDelay = 60000; // 60 seconds - return Math.min(baseDelay * Math.pow(2, retryCount), maxDelay); + return calculateExponentialBackoff(retryCount, 5000, 60000); }; /** diff --git a/src/processors/shared/delayed.producer.ts b/src/processors/shared/delayed.producer.ts index 584c898..533a05b 100644 --- a/src/processors/shared/delayed.producer.ts +++ b/src/processors/shared/delayed.producer.ts @@ -6,6 +6,8 @@ import { Producer, Partitioners } from 'kafkajs'; import { kafka } from '@src/config/kafka.config.js'; import { CORE_TOPICS, getTopicForChannel, type delayed_notification_topic } from '@src/types/types.js'; +import { calculateExponentialBackoff } from '@src/utils/backoff.utils.js'; + let producer: Producer | null = null; /** @@ -28,9 +30,7 @@ export const initDelayedProducer = async (): Promise => { * Calculate exponential backoff delay in milliseconds */ const calculateBackoffDelay = (retryCount: number): number => { - const baseDelay = 1000; // 1 second - const maxDelay = 300000; // 5 minutes - return Math.min(baseDelay * Math.pow(2, retryCount), maxDelay); + return calculateExponentialBackoff(retryCount, 1000, 300000); }; /** diff --git a/src/processors/shared/rate-limiter.ts b/src/processors/shared/rate-limiter.ts index dadb1e0..92a0665 100644 --- a/src/processors/shared/rate-limiter.ts +++ b/src/processors/shared/rate-limiter.ts @@ -6,6 +6,7 @@ */ import { getRedisClient } from '@src/config/redis.config.js'; +import { env } from '@src/config/env.config.js'; import { getRateLimitConfig as getPluginRateLimitConfig } from '@src/plugins/index.js'; import { rateLimiterLogger as logger } from '@src/workers/utils/logger.js'; import type { RateLimitConfig, RefillInterval, RateLimitResult } from '@src/types/types.js'; @@ -66,32 +67,30 @@ const buildKeys = (providerId: string): { tokensKey: string; lastRefillKey: stri }; }; -/** - * Token Bucket Algorithm Result - */ /** * Try to consume a token from the bucket - * Uses Redis Lua script for atomic operation with queue-based staggering + * Uses Redis Lua script for atomic operation with fixed retry delay */ export const consumeToken = async (providerId: string): Promise => { const redis = getRedisClient(); const config = getConfig(providerId); const normalizedRate = normalizeRefillRate(config); - const { tokensKey, lastRefillKey, queueKey } = buildKeys(providerId); + const { tokensKey, lastRefillKey } = buildKeys(providerId); + const fixedRetryDelay = env.RATE_LIMIT_RETRY_DELAY_MS; // Debug logging - logger.debug(`Provider: ${providerId}, Config: maxTokens=${config.maxTokens}, refillRate=${config.refillRate}/${config.refillInterval || 'second'} (normalized: ${normalizedRate.toFixed(6)}/sec)`); + logger.debug(`Provider: ${providerId}, Config: maxTokens=${config.maxTokens}, refillRate=${config.refillRate}/${config.refillInterval || 'second'} (normalized: ${normalizedRate.toFixed(6)}/sec), retryDelay=${fixedRetryDelay}ms`); const now = Date.now(); - // Lua script for atomic token bucket operation with queue position tracking + // Lua script for atomic token bucket operation with fixed retry delay const luaScript = ` local tokens_key = KEYS[1] local last_refill_key = KEYS[2] - local queue_key = KEYS[3] local max_tokens = tonumber(ARGV[1]) local refill_rate = tonumber(ARGV[2]) local now = tonumber(ARGV[3]) + local fixed_retry_delay = tonumber(ARGV[4]) -- Get current state local current_tokens = tonumber(redis.call('GET', tokens_key)) or max_tokens @@ -107,41 +106,29 @@ export const consumeToken = async (providerId: string): Promise new_tokens = new_tokens - 1 redis.call('SET', tokens_key, new_tokens) redis.call('SET', last_refill_key, now) - -- Reset queue position when tokens are available (batch consumed) - redis.call('SET', queue_key, 0) - redis.call('EXPIRE', queue_key, 86400) - return { 1, new_tokens, 0, 0 } -- allowed, remaining, wait_time, queue_position + return { 1, new_tokens, 0 } else - -- Rate limited: calculate staggered delay based on queue position - local time_per_token = 1000 / refill_rate -- ms per token - local queue_position = redis.call('INCR', queue_key) - 1 - redis.call('EXPIRE', queue_key, 86400) -- 24h TTL - - -- Staggered delay: (position + 1) × time_per_token - local staggered_delay = (queue_position + 1) * time_per_token - - return { 0, new_tokens, staggered_delay, queue_position } -- denied, remaining, wait_time, queue_position + return { 0, new_tokens, fixed_retry_delay } end `; const result = await redis.eval( luaScript, - 3, // 3 keys now + 2, tokensKey, lastRefillKey, - queueKey, config.maxTokens.toString(), normalizedRate.toString(), - now.toString() - ) as [number, number, number, number]; + now.toString(), + fixedRetryDelay.toString() + ) as [number, number, number]; - const [allowed, remainingTokens, retryAfterMs, queuePosition] = result; + const [allowed, remainingTokens, retryAfterMs] = result; return { allowed: allowed === 1, remainingTokens: Math.floor(remainingTokens), retryAfterMs: retryAfterMs > 0 ? Math.ceil(retryAfterMs) : undefined, - queuePosition: allowed === 0 ? queuePosition : undefined }; }; diff --git a/src/processors/unified/unified.consumer.ts b/src/processors/unified/unified.consumer.ts index a285f4c..1b006ce 100644 --- a/src/processors/unified/unified.consumer.ts +++ b/src/processors/unified/unified.consumer.ts @@ -20,12 +20,14 @@ import { unifiedProcessorLogger as logger } from './unified.logger.js'; import { sendWithFallback, PluginRegistry, + getProviderCascade, resolveFallbackProviderId, validateNotificationForProvider } from '@src/plugins/index.js'; import type { BaseNotification, DeliveryResult, + RateLimitResult, ConsumerHealthState, ConsumerHealthCheckResult, ConsumerHealthDetails @@ -119,7 +121,8 @@ export const scheduleFallbackProviderHandoff = async ( channel: string, notification: BaseNotification, currentProviderId: string | undefined, - errorMessage: string + errorMessage: string, + delayMs?: number ): Promise => { const fallbackProviderId = resolveFallbackProviderId(channel, currentProviderId); @@ -145,17 +148,29 @@ export const scheduleFallbackProviderHandoff = async ( } await setFailed(notification.notification_id.toString(), notification.retry_count); - const delayedPayload = buildDelayedPayloadGeneric( - validationResult.data as unknown as Record, - channel, - 0 - ); + const delayedPayload = delayMs !== undefined + ? buildDelayedPayloadGeneric( + validationResult.data as unknown as Record, + channel, + 0, + delayMs + ) + : buildDelayedPayloadGeneric( + validationResult.data as unknown as Record, + channel, + 0 + ); // eslint-disable-next-line @typescript-eslint/no-explicit-any await publishDelayed(delayedPayload as any); + const isRateLimit = errorMessage.toLowerCase().includes('rate limit'); + const alertTitle = isRateLimit + ? `INFO: RATE LIMIT EXCEEDED, SWITCHING TO FALLBACK\n` + : `INFO: RETRY BUDGET EXHAUSTED, SWITCHING TO FALLBACK\n`; + void AdminAlertService.sendAlert( 'service_health', - `INFO: RETRY BUDGET EXHAUSTED, SWITCHING TO FALLBACK\n` + + alertTitle + `Notification ID: ${notification.notification_id}\n` + `Channel: ${channel}\n` + `Primary provider: ${currentProviderId || 'default'}\n` + @@ -166,8 +181,7 @@ export const scheduleFallbackProviderHandoff = async ( ); logger.warn( - `[${channel}] Retry budget exhausted for provider ${currentProviderId || 'default'}, ` + - `scheduled fallback handoff to ${fallbackProviderId}: ${notification.notification_id}` + `[${channel}] Fallback handoff scheduled for provider ${currentProviderId || 'default'} to ${fallbackProviderId}: ${notification.notification_id}` ); return true; }; @@ -273,34 +287,50 @@ const processMessage = async ( logger.info(`[${channel}] Retrying previously failed: ${notificationId}`); } - // 4. Rate limit check - uses provider ID for per-provider rate limiting - const rateLimitResult = await consumeToken(providerId!); + // 4. Rate limit check across the provider cascade + const cascade = getProviderCascade(channel, rawData.provider); + let selectedProviderId: string | null = null; + let lastRateLimitResult: RateLimitResult | null = null; + + for (let i = 0; i < cascade.length; i++) { + const candidateId = cascade[i]; + const rateLimitResult = await consumeToken(candidateId); + + if (rateLimitResult.allowed) { + selectedProviderId = candidateId; + if (i > 0) { + logger.info(`[${channel}] Cascade failover: '${cascade[0]}' rate-limited, using fallback '${candidateId}': ${notificationId}`); + void AdminAlertService.sendAlert('service_health', + `ℹ️ RATE LIMIT CASCADE: USING FALLBACK (${i + 1}/${cascade.length})\n` + + `Channel: ${channel}\n` + + `Notification ID: ${notificationId}\n` + + `Rate-limited providers: ${cascade.slice(0, i).join(', ')}\n` + + `Active fallback: ${candidateId}\n` + + `Action: Check primary provider rate limits if this persists.`, + { severity: 'info', notificationId, channel }); + } + break; + } else { + lastRateLimitResult = rateLimitResult; + logger.warn(`[${channel}] Provider '${candidateId}' rate limited (${i + 1}/${cascade.length}): ${notificationId}`); + } + } - if (!rateLimitResult.allowed) { - logger.warn(`[${channel}] Rate limited: ${notificationId}, retry after ${rateLimitResult.retryAfterMs}ms`); + if (!selectedProviderId) { + // All providers in cascade are rate-limited + logger.warn(`[${channel}] All providers in cascade rate-limited: ${notificationId}`); const newRetryCount = notification.retry_count + 1; if (newRetryCount > env.MAX_RETRY_COUNT) { const currentProviderId = notification.provider || PluginRegistry.getDefaultProviderId(channel); - const handoffScheduled = await scheduleFallbackProviderHandoff( - channel, - notification, - currentProviderId, - `Rate limited after ${env.MAX_RETRY_COUNT} retries` - ); - - if (handoffScheduled) { - return true; - } - - logger.error(`[${channel}] Max retries exceeded (rate limited): ${notificationId}`); + logger.error(`[${channel}] Max retries exceeded (all cascade providers rate limited): ${notificationId}`); void AdminAlertService.sendAlert('failed_notification', `❌ MAX RETRIES EXCEEDED (RATE LIMITED)\n` + `Notification ID: ${notificationId}\n` + `Channel: ${channel}\n` + `Provider: ${currentProviderId || providerId || 'default'}\n` + - `Root cause: Provider rate limit exhausted after ${env.MAX_RETRY_COUNT} retries\n` + + `Root cause: All cascade providers rate limit exhausted after ${env.MAX_RETRY_COUNT} retries\n` + `Action: Check provider rate limits in simplens.config.yaml. Consider increasing limits or adding fallback provider.`, { severity: 'critical', notificationId, channel }); @@ -309,20 +339,26 @@ const processMessage = async ( return true; } - // Push to delayed queue using rate limiter's retryAfterMs + // Push to delayed queue using fixed retry delay from rate limiter await setRateLimited(notificationId, validationResult.data.retry_count); + const retryDelay = lastRateLimitResult?.retryAfterMs ?? env.RATE_LIMIT_RETRY_DELAY_MS; const delayedPayload = buildDelayedPayloadGeneric( notification as unknown as Record, channel, newRetryCount, - rateLimitResult.retryAfterMs // Use exact delay from rate limiter + retryDelay ); // eslint-disable-next-line @typescript-eslint/no-explicit-any await publishDelayed(delayedPayload as any); - logger.info(`[${channel}] Rate limited, scheduled retry in ${rateLimitResult.retryAfterMs}ms: ${notificationId} (retry ${newRetryCount})`); + logger.info(`[${channel}] Rate limited, scheduled retry in ${retryDelay}ms: ${notificationId} (retry ${newRetryCount})`); return true; } + // If selected fallback provider differs from original provider, update notification provider + if (selectedProviderId !== notification.provider) { + notification.provider = selectedProviderId; + } + // 5. Send via plugin router (with auto-fallback) const result: DeliveryResult = await sendWithFallback(channel, notification); diff --git a/src/types/schemas.ts b/src/types/schemas.ts index 9c0833e..7353cf5 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -514,6 +514,7 @@ export const operationalSettingsSchema = z.object({ max_retry_count: z.number().int().min(1).max(30).default(5), idempotency_ttl_seconds: z.number().int().min(60).max(2592000).default(86400), processing_ttl_seconds: z.number().int().min(10).max(1800).default(120), + rate_limit_retry_delay_ms: z.number().int().min(100).max(60000).default(5000), }), delayed: z.object({ delayed_poll_interval_ms: z.number().int().min(200).max(30000).default(1000), diff --git a/src/utils/backoff.utils.ts b/src/utils/backoff.utils.ts new file mode 100644 index 0000000..89e2322 --- /dev/null +++ b/src/utils/backoff.utils.ts @@ -0,0 +1,22 @@ +/** + * Backoff calculation utilities + */ + +/** + * Calculate exponential backoff delay in milliseconds + * Formula: Math.min(baseDelayMs * Math.pow(2, retryCount), maxDelayMs) + * + * @param retryCount - Current retry attempt count (>= 0) + * @param baseDelayMs - Base delay in milliseconds (default: 1000ms) + * @param maxDelayMs - Maximum delay cap in milliseconds (default: 300000ms / 5 minutes) + * @returns Delay in milliseconds + */ +export function calculateExponentialBackoff( + retryCount: number, + baseDelayMs = 1000, + maxDelayMs = 300000 +): number { + const safeRetryCount = Math.max(0, retryCount); + const delay = baseDelayMs * Math.pow(2, safeRetryCount); + return Math.min(delay, maxDelayMs); +} diff --git a/tests/unit/config/dynamic-config.test.ts b/tests/unit/config/dynamic-config.test.ts index 45b47f4..b4e38e8 100644 --- a/tests/unit/config/dynamic-config.test.ts +++ b/tests/unit/config/dynamic-config.test.ts @@ -112,13 +112,18 @@ describe('Dynamic Operational Configuration', () => { worker: { outbox_batch_size: 42, }, + retry: { + rate_limit_retry_delay_ms: 3000, + }, logging: { log_level: 'debug', }, }); expect(dynamicConfig.get('worker').outbox_batch_size).toBe(42); + expect(dynamicConfig.get('retry').rate_limit_retry_delay_ms).toBe(3000); expect(env.OUTBOX_BATCH_SIZE).toBe(42); + expect(env.RATE_LIMIT_RETRY_DELAY_MS).toBe(3000); expect(env.LOG_LEVEL).toBe('debug'); expect(changeFired).toBe(true); @@ -128,7 +133,9 @@ describe('Dynamic Operational Configuration', () => { it('should reset settings to system defaults', async () => { await dynamicConfig.resetSettings(); expect(dynamicConfig.get('worker').outbox_batch_size).toBe(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS.worker.outbox_batch_size); + expect(dynamicConfig.get('retry').rate_limit_retry_delay_ms).toBe(5000); expect(env.OUTBOX_BATCH_SIZE).toBe(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS.worker.outbox_batch_size); + expect(env.RATE_LIMIT_RETRY_DELAY_MS).toBe(5000); expect(env.LOG_LEVEL).toBe(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS.logging.log_level); }); }); diff --git a/tests/unit/plugins/router.test.ts b/tests/unit/plugins/router.test.ts index 6340810..0c21ec9 100644 --- a/tests/unit/plugins/router.test.ts +++ b/tests/unit/plugins/router.test.ts @@ -26,6 +26,10 @@ vi.mock('../../../src/processors/unified/unified.logger.js', () => ({ }, })); +vi.mock('../../../src/processors/shared/rate-limiter.js', () => ({ + consumeToken: vi.fn().mockResolvedValue({ allowed: true }), +})); + // Create a mock provider factory function createMockProvider(options: { name: string; @@ -72,6 +76,7 @@ describe('Provider Router', () => { let validateNotification: typeof import('../../../src/plugins/loader/router.js').validateNotification; let validateNotificationForProvider: typeof import('../../../src/plugins/loader/router.js').validateNotificationForProvider; let resolveFallbackProviderId: typeof import('../../../src/plugins/loader/router.js').resolveFallbackProviderId; + let getProviderCascade: typeof import('../../../src/plugins/loader/router.js').getProviderCascade; let getRateLimitConfig: typeof import('../../../src/plugins/loader/router.js').getRateLimitConfig; beforeEach(async () => { @@ -85,6 +90,7 @@ describe('Provider Router', () => { validateNotification = routerModule.validateNotification; validateNotificationForProvider = routerModule.validateNotificationForProvider; resolveFallbackProviderId = routerModule.resolveFallbackProviderId; + getProviderCascade = routerModule.getProviderCascade; getRateLimitConfig = routerModule.getRateLimitConfig; PluginRegistry.clear(); @@ -378,6 +384,35 @@ describe('Provider Router', () => { }); }); + describe('getProviderCascade', () => { + it('should return full cascade of primary and fallback providers', () => { + const primary = createMockProvider({ name: 'primary', channel: 'email' }); + const fb1 = createMockProvider({ name: 'fb1', channel: 'email' }); + const fb2 = createMockProvider({ name: 'fb2', channel: 'email' }); + + PluginRegistry.register(primary as any, 'primary', 3); + PluginRegistry.register(fb1 as any, 'fb1', 2); + PluginRegistry.register(fb2 as any, 'fb2', 1); + PluginRegistry.setChannelConfig('email', { default: 'primary', fallback: ['fb1', 'fb2'] }); + + expect(getProviderCascade('email')).toEqual(['primary', 'fb1', 'fb2']); + }); + + it('should return cascade starting from currentProviderId onwards', () => { + const primary = createMockProvider({ name: 'primary', channel: 'email' }); + const fb1 = createMockProvider({ name: 'fb1', channel: 'email' }); + const fb2 = createMockProvider({ name: 'fb2', channel: 'email' }); + + PluginRegistry.register(primary as any, 'primary', 3); + PluginRegistry.register(fb1 as any, 'fb1', 2); + PluginRegistry.register(fb2 as any, 'fb2', 1); + PluginRegistry.setChannelConfig('email', { default: 'primary', fallback: ['fb1', 'fb2'] }); + + expect(getProviderCascade('email', 'fb1')).toEqual(['fb1', 'fb2']); + expect(getProviderCascade('email', 'fb2')).toEqual(['fb2']); + }); + }); + describe('getRateLimitConfig', () => { it('should return rate limit config for provider', () => { const gmail = createMockProvider({ diff --git a/tests/unit/processors/unified.consumer.test.ts b/tests/unit/processors/unified.consumer.test.ts index 42fa451..339e995 100644 --- a/tests/unit/processors/unified.consumer.test.ts +++ b/tests/unit/processors/unified.consumer.test.ts @@ -10,6 +10,7 @@ const publishDelayedMock = vi.fn().mockResolvedValue(undefined); const publishStatusMock = vi.fn().mockResolvedValue(undefined); const consumeTokenMock = vi.fn(); const resolveFallbackProviderIdMock = vi.fn(); +const getProviderCascadeMock = vi.fn(); const validateNotificationForProviderMock = vi.fn(); const sendWithFallbackMock = vi.fn(); @@ -59,6 +60,7 @@ vi.mock('../../../src/plugins/index.js', () => ({ getDefaultProviderId: pluginRegistryGetDefaultProviderIdMock, }, sendWithFallback: sendWithFallbackMock, + getProviderCascade: getProviderCascadeMock, resolveFallbackProviderId: resolveFallbackProviderIdMock, validateNotificationForProvider: validateNotificationForProviderMock, })); @@ -135,6 +137,9 @@ describe('scheduleFallbackProviderHandoff', () => { }); tryAcquireProcessingLockMock.mockResolvedValue({ canProcess: true, isRetry: false }); consumeTokenMock.mockResolvedValue({ allowed: true }); + getProviderCascadeMock.mockImplementation((_channel: string, currentProviderId?: string) => { + return currentProviderId ? [currentProviderId] : ['primary']; + }); sendWithFallbackMock.mockResolvedValue({ success: true, messageId: 'msg-1' }); buildDelayedPayloadGenericMock.mockReturnValue({ @@ -287,6 +292,21 @@ describe('processMessage via consumer run loop', () => { await stopUnifiedConsumer('email'); }; + beforeEach(() => { + vi.clearAllMocks(); + sendWithFallbackMock.mockResolvedValue({ success: true, messageId: 'msg-1' }); + consumeTokenMock.mockResolvedValue({ allowed: true }); + getProviderCascadeMock.mockImplementation((_channel: string, currentProviderId?: string) => { + return currentProviderId ? [currentProviderId] : ['primary']; + }); + resolveFallbackProviderIdMock.mockReturnValue(undefined); + tryAcquireProcessingLockMock.mockResolvedValue({ canProcess: true, isRetry: false }); + providerSafeParseMock.mockImplementation((notification) => ({ + success: true, + data: notification, + })); + }); + it('should set failed, publish failure status, and alert for non-retryable provider errors', async () => { sendWithFallbackMock.mockResolvedValue({ success: false, @@ -313,34 +333,52 @@ describe('processMessage via consumer run loop', () => { expect(commitOffsetsMock).toHaveBeenCalledTimes(1); }); - it('should hand off to fallback when rate-limit retry budget is exceeded', async () => { - consumeTokenMock.mockResolvedValue({ allowed: false, retryAfterMs: 1000 }); - resolveFallbackProviderIdMock.mockReturnValue('fallback'); - validateNotificationForProviderMock.mockImplementation((_providerId, notification) => ({ - success: true, - data: notification, - })); + it('should cascade to fallback provider in real time when primary is rate-limited and deliver immediately', async () => { + getProviderCascadeMock.mockReturnValue(['primary', 'fallback']); + consumeTokenMock.mockImplementation(async (providerId) => { + if (providerId === 'primary') { + return { allowed: false, retryAfterMs: 5000 }; + } + return { allowed: true, remainingTokens: 10 }; + }); - await startAndProcess(createNotification({ retry_count: Number.MAX_SAFE_INTEGER })); + await startAndProcess(createNotification()); - expect(resolveFallbackProviderIdMock).toHaveBeenCalledWith('email', 'primary'); + expect(consumeTokenMock).toHaveBeenCalledWith('primary'); + expect(consumeTokenMock).toHaveBeenCalledWith('fallback'); + expect(sendWithFallbackMock).toHaveBeenCalledWith( + 'email', + expect.objectContaining({ provider: 'fallback' }) + ); + expect(setDeliveredMock).toHaveBeenCalledWith('notif-123', 0); + expect(commitOffsetsMock).toHaveBeenCalledTimes(1); + }); + + it('should push to delayed queue with fixed delay when all cascade providers are rate limited', async () => { + getProviderCascadeMock.mockReturnValue(['primary', 'fallback']); + consumeTokenMock.mockResolvedValue({ allowed: false, retryAfterMs: 5000 }); + + await startAndProcess(createNotification({ retry_count: 0 })); + + expect(consumeTokenMock).toHaveBeenCalledWith('primary'); + expect(consumeTokenMock).toHaveBeenCalledWith('fallback'); + expect(setRateLimitedMock).toHaveBeenCalledWith('notif-123', 0); expect(buildDelayedPayloadGenericMock).toHaveBeenCalledWith( - expect.objectContaining({ provider: 'fallback' }), + expect.anything(), 'email', - 0 + 1, + 5000 ); expect(publishDelayedMock).toHaveBeenCalledTimes(1); - expect(publishStatusMock).not.toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); expect(commitOffsetsMock).toHaveBeenCalledTimes(1); }); - it('should set failed when retry budget exceeded and current provider is fallback', async () => { - consumeTokenMock.mockResolvedValue({ allowed: false, retryAfterMs: 1000 }); - resolveFallbackProviderIdMock.mockReturnValue(undefined); + it('should set failed when retry budget exceeded and all cascade providers are rate-limited', async () => { + getProviderCascadeMock.mockReturnValue(['primary', 'fallback']); + consumeTokenMock.mockResolvedValue({ allowed: false, retryAfterMs: 5000 }); - await startAndProcess(createNotification({ provider: 'fallback', retry_count: Number.MAX_SAFE_INTEGER })); + await startAndProcess(createNotification({ retry_count: Number.MAX_SAFE_INTEGER })); - expect(resolveFallbackProviderIdMock).toHaveBeenCalledWith('email', 'fallback'); expect(setFailedMock).toHaveBeenCalledWith('notif-123', Number.MAX_SAFE_INTEGER); expect(publishStatusMock).toHaveBeenCalledWith(expect.objectContaining({ notification_id: 'notif-123', diff --git a/tests/unit/shared/rate-limiter.test.ts b/tests/unit/shared/rate-limiter.test.ts index 19a99ab..6900af4 100644 --- a/tests/unit/shared/rate-limiter.test.ts +++ b/tests/unit/shared/rate-limiter.test.ts @@ -21,6 +21,7 @@ vi.mock('../../../src/config/redis.config.js', () => ({ vi.mock('../../../src/config/env.config.js', () => ({ env: { + RATE_LIMIT_RETRY_DELAY_MS: 5000, EMAIL_RATE_LIMIT_TOKENS: 100, EMAIL_RATE_LIMIT_REFILL_RATE: 10, WHATSAPP_RATE_LIMIT_TOKENS: 50, @@ -37,8 +38,8 @@ describe('Rate Limiter Module', () => { it('should return allowed: true when tokens are available', async () => { const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - // Mock: Lua script returns [1, 99, 0, 0] meaning allowed with 99 remaining - mockRedisClient.eval.mockResolvedValue([1, 99, 0, 0]); + // Mock: Lua script returns [1, 99, 0] meaning allowed with 99 remaining + mockRedisClient.eval.mockResolvedValue([1, 99, 0]); const result = await consumeToken('email'); @@ -47,48 +48,46 @@ describe('Rate Limiter Module', () => { expect(result.retryAfterMs).toBeUndefined(); }); - it('should return allowed: false when rate limited', async () => { + it('should return allowed: false with fixed retry delay when rate limited', async () => { const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - // Mock: Lua script returns [0, 0, 100, 0] meaning denied, 0 tokens, retry in 100ms, queue position 0 - mockRedisClient.eval.mockResolvedValue([0, 0, 100, 0]); + // Mock: Lua script returns [0, 0, 5000] meaning denied, 0 tokens, retry in 5000ms + mockRedisClient.eval.mockResolvedValue([0, 0, 5000]); const result = await consumeToken('email'); expect(result.allowed).toBe(false); expect(result.remainingTokens).toBe(0); - expect(result.retryAfterMs).toBe(100); + expect(result.retryAfterMs).toBe(5000); }); it('should use email config for email channel', async () => { const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.eval.mockResolvedValue([1, 99, 0, 0]); + mockRedisClient.eval.mockResolvedValue([1, 99, 0]); await consumeToken('email'); // Verify the Lua script was called with email configuration - // eval(script, numKeys, key1, key2, arg1, arg2, arg3) - // arg1=maxTokens, arg2=refillRate, arg3=now + // eval(script, numKeys=2, key1, key2, arg1=maxTokens, arg2=refillRate, arg3=now, arg4=fixedRetryDelay) expect(mockRedisClient.eval).toHaveBeenCalled(); const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) - expect(callArgs[5]).toBe('100'); // maxTokens for email (ARGV[1]) - expect(callArgs[6]).toBe('10'); // refillRate for email (ARGV[2]) + expect(callArgs[4]).toBe('100'); // maxTokens for email (ARGV[1]) + expect(callArgs[5]).toBe('10'); // refillRate for email (ARGV[2]) + expect(callArgs[7]).toBe('5000'); // fixed retry delay (ARGV[4]) }); it('should use default config for whatsapp channel', async () => { const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.eval.mockResolvedValue([1, 99, 0, 0]); + mockRedisClient.eval.mockResolvedValue([1, 99, 0]); await consumeToken('whatsapp'); const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) // Plugin system defaults all channels to same config now - expect(callArgs[5]).toBe('100'); // maxTokens (default) - expect(callArgs[6]).toBe('10'); // refillRate (default) + expect(callArgs[4]).toBe('100'); // maxTokens (default) + expect(callArgs[5]).toBe('10'); // refillRate (default) }); }); @@ -101,7 +100,7 @@ describe('Rate Limiter Module', () => { const count = await getTokenCount('email'); - expect(count).toBe(100); // EMAIL_RATE_LIMIT_TOKENS + expect(count).toBe(100); }); it('should return current token count from Redis', async () => { @@ -113,7 +112,6 @@ describe('Rate Limiter Module', () => { const count = await getTokenCount('email'); - // Should be approximately 50 (may have slight variation due to timing) expect(count).toBeGreaterThanOrEqual(50); expect(count).toBeLessThanOrEqual(100); }); @@ -127,7 +125,6 @@ describe('Rate Limiter Module', () => { const count = await getTokenCount('email'); - // Should be 50 + 10 = 60 (approximately) expect(count).toBeGreaterThanOrEqual(59); expect(count).toBeLessThanOrEqual(61); }); @@ -150,13 +147,11 @@ describe('Rate Limiter Module', () => { }); describe('refillInterval normalization', () => { - // Mock the plugin registry to return different interval configs beforeEach(() => { vi.resetModules(); }); it('should normalize refillInterval: minute to per-second rate', async () => { - // Mock plugin to return minute config vi.doMock('../../../src/plugins/index.js', () => ({ getRateLimitConfig: vi.fn().mockReturnValue({ maxTokens: 60, @@ -166,14 +161,13 @@ describe('Rate Limiter Module', () => { })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.eval.mockResolvedValue([1, 59, 0, 0]); + mockRedisClient.eval.mockResolvedValue([1, 59, 0]); await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) - expect(callArgs[5]).toBe('60'); // maxTokens - expect(callArgs[6]).toBe('1'); // 60/minute = 1/second + expect(callArgs[4]).toBe('60'); // maxTokens + expect(callArgs[5]).toBe('1'); // 60/minute = 1/second }); it('should normalize refillInterval: hour to per-second rate', async () => { @@ -186,14 +180,13 @@ describe('Rate Limiter Module', () => { })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.eval.mockResolvedValue([1, 3599, 0, 0]); + mockRedisClient.eval.mockResolvedValue([1, 3599, 0]); await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) - expect(callArgs[5]).toBe('3600'); // maxTokens - expect(callArgs[6]).toBe('1'); // 3600/hour = 1/second + expect(callArgs[4]).toBe('3600'); // maxTokens + expect(callArgs[5]).toBe('1'); // 3600/hour = 1/second }); it('should normalize refillInterval: day to per-second rate', async () => { @@ -206,15 +199,13 @@ describe('Rate Limiter Module', () => { })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.eval.mockResolvedValue([1, 499, 0, 0]); + mockRedisClient.eval.mockResolvedValue([1, 499, 0]); await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) - expect(callArgs[5]).toBe('500'); // maxTokens - // 500/day = 500/86400 ≈ 0.005787 per second - const normalizedRate = parseFloat(callArgs[6]); + expect(callArgs[4]).toBe('500'); // maxTokens + const normalizedRate = parseFloat(callArgs[5]); expect(normalizedRate).toBeCloseTo(500 / 86400, 5); }); @@ -223,19 +214,17 @@ describe('Rate Limiter Module', () => { getRateLimitConfig: vi.fn().mockReturnValue({ maxTokens: 100, refillRate: 10 - // No refillInterval specified }) })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.eval.mockResolvedValue([1, 99, 0, 0]); + mockRedisClient.eval.mockResolvedValue([1, 99, 0]); await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) - expect(callArgs[5]).toBe('100'); // maxTokens - expect(callArgs[6]).toBe('10'); // 10/second (no normalization needed) + expect(callArgs[4]).toBe('100'); // maxTokens + expect(callArgs[5]).toBe('10'); // 10/second }); }); }); diff --git a/tests/unit/utils/backoff.utils.test.ts b/tests/unit/utils/backoff.utils.test.ts new file mode 100644 index 0000000..907efa3 --- /dev/null +++ b/tests/unit/utils/backoff.utils.test.ts @@ -0,0 +1,29 @@ +import { describe, it, expect } from 'vitest'; +import { calculateExponentialBackoff } from '../../../src/utils/backoff.utils.js'; + +describe('calculateExponentialBackoff', () => { + it('should calculate exponential backoff with default base and max delays', () => { + expect(calculateExponentialBackoff(0)).toBe(1000); // 1000 * 2^0 + expect(calculateExponentialBackoff(1)).toBe(2000); // 1000 * 2^1 + expect(calculateExponentialBackoff(2)).toBe(4000); // 1000 * 2^2 + expect(calculateExponentialBackoff(3)).toBe(8000); // 1000 * 2^3 + expect(calculateExponentialBackoff(4)).toBe(16000); // 1000 * 2^4 + expect(calculateExponentialBackoff(5)).toBe(32000); // 1000 * 2^5 + }); + + it('should handle custom baseDelay and maxDelay', () => { + expect(calculateExponentialBackoff(0, 5000, 60000)).toBe(5000); + expect(calculateExponentialBackoff(1, 5000, 60000)).toBe(10000); + expect(calculateExponentialBackoff(2, 5000, 60000)).toBe(20000); + expect(calculateExponentialBackoff(3, 5000, 60000)).toBe(40000); + expect(calculateExponentialBackoff(4, 5000, 60000)).toBe(60000); // capped at 60s + }); + + it('should cap at maxDelayMs for high retry counts', () => { + expect(calculateExponentialBackoff(20, 1000, 300000)).toBe(300000); + }); + + it('should handle negative retry counts gracefully', () => { + expect(calculateExponentialBackoff(-1)).toBe(1000); + }); +}); From 09e412b7e3dc24654b87c16d5ca161a7e0272a0f Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Mon, 21 Sep 2026 10:28:55 +0530 Subject: [PATCH 25/46] feat: implement provider execution history tracking and update notification models --- dashboard/app/events/[id]/page.tsx | 82 ++++++++++ dashboard/lib/types.ts | 12 ++ .../notifications-management.controller.ts | 4 + src/database/models/notification.models.ts | 12 ++ src/database/models/status-outbox.models.ts | 15 ++ src/plugins/loader/router.ts | 146 ++++++++++++++++-- src/processors/unified/unified.consumer.ts | 30 +++- src/types/schemas.ts | 16 ++ src/types/types.ts | 5 + src/workers/consumers/status.consumer.ts | 8 + src/workers/producers/background.producer.ts | 2 + .../workers/consumers/status.consumer.test.ts | 40 +++++ tests/unit/plugins/router.test.ts | 35 +++++ .../unit/processors/unified.consumer.test.ts | 12 ++ 14 files changed, 404 insertions(+), 15 deletions(-) diff --git a/dashboard/app/events/[id]/page.tsx b/dashboard/app/events/[id]/page.tsx index df5f451..6c39029 100644 --- a/dashboard/app/events/[id]/page.tsx +++ b/dashboard/app/events/[id]/page.tsx @@ -382,6 +382,88 @@ export default function EventDetailPage({ params }: PageProps) { + {/* Provider Execution History */} + + +
+
+ Provider Execution History + + Timeline of provider dispatch attempts, rate limit cascades, and failure reasons + +
+ {notification.provider_history && notification.provider_history.length > 0 && ( + + {notification.provider_history.length} {notification.provider_history.length === 1 ? 'attempt' : 'attempts'} + + )} +
+ + + {(!notification.provider_history || notification.provider_history.length === 0) ? ( +
+ No intermediate provider cascade attempts recorded. +
+ ) : ( +
+ {notification.provider_history.map((attempt, index) => { + const isSuccess = attempt.status === 'delivered'; + const isRateLimit = attempt.status === 'rate_limited'; + + return ( +
+
+
+ {index + 1}. {attempt.provider} + + {isSuccess ? 'Delivered' : isRateLimit ? 'Rate Limited' : 'Failed'} + + {attempt.error_code && ( + + {attempt.error_code} + + )} + {attempt.duration_ms !== undefined && ( + + ({attempt.duration_ms}ms) + + )} +
+ {attempt.error_message && ( +

+ {attempt.error_message} +

+ )} +
+
+ + {format(new Date(attempt.attempted_at), "PPpp")} +
+
+ ); + })} +
+ )} + + + {/* Error Message (if failed) */} {isFailed && notification.error_message && ( diff --git a/dashboard/lib/types.ts b/dashboard/lib/types.ts index 2204fe0..267e005 100644 --- a/dashboard/lib/types.ts +++ b/dashboard/lib/types.ts @@ -22,6 +22,17 @@ export interface Recipient { // Generic content - structure depends on channel export type NotificationContent = Record; +export interface ProviderAttempt { + provider: string; + status: 'delivered' | 'failed' | 'rate_limited'; + error_code?: string; + error_message?: string; + retryable?: boolean; + attempted_at: Date | string; + duration_ms?: number; + response?: unknown; +} + export interface Notification { _id: string; request_id: string; @@ -29,6 +40,7 @@ export interface Notification { client_name?: string; channel: Channel; provider?: string; // Provider ID used for this notification + provider_history?: ProviderAttempt[]; recipient: Recipient; content: NotificationContent; variables?: Record; diff --git a/src/api/controllers/notifications-management.controller.ts b/src/api/controllers/notifications-management.controller.ts index 5f7b030..61e61c5 100644 --- a/src/api/controllers/notifications-management.controller.ts +++ b/src/api/controllers/notifications-management.controller.ts @@ -104,6 +104,7 @@ export const listNotifications = async (req: Request, res: Response): Promise( type: String, index: true, }, + provider_history: [ + { + provider: { type: String, required: true }, + status: { type: String, enum: ['delivered', 'failed', 'rate_limited'], required: true }, + error_code: { type: String }, + error_message: { type: String }, + retryable: { type: Boolean }, + attempted_at: { type: Date, default: Date.now }, + duration_ms: { type: Number }, + response: { type: mongoose.Schema.Types.Mixed }, + } + ], // Dynamic recipient schema - structure depends on channel recipient: { type: mongoose.Schema.Types.Mixed, diff --git a/src/database/models/status-outbox.models.ts b/src/database/models/status-outbox.models.ts index 38d361d..4dcd8da 100644 --- a/src/database/models/status-outbox.models.ts +++ b/src/database/models/status-outbox.models.ts @@ -21,6 +21,21 @@ const status_outbox_schema = new mongoose.Schema( enum: Object.values(NOTIFICATION_STATUS_SF), required: true, }, + provider: { + type: String, + }, + provider_history: [ + { + provider: { type: String, required: true }, + status: { type: String, enum: ['delivered', 'failed', 'rate_limited'], required: true }, + error_code: { type: String }, + error_message: { type: String }, + retryable: { type: Boolean }, + attempted_at: { type: Date, default: Date.now }, + duration_ms: { type: Number }, + response: { type: mongoose.Schema.Types.Mixed }, + } + ], processed: { type: Boolean, default: false, diff --git a/src/plugins/loader/router.ts b/src/plugins/loader/router.ts index 8f5cb1b..e0ec2d0 100644 --- a/src/plugins/loader/router.ts +++ b/src/plugins/loader/router.ts @@ -5,7 +5,7 @@ */ import { PluginRegistry } from './registry.js'; -import type { DeliveryResult, BaseNotification } from '@src/types/types.js'; +import type { DeliveryResult, BaseNotification, ProviderAttempt } from '@src/types/types.js'; import { unifiedProcessorLogger as logger } from '@src/processors/unified/unified.logger.js'; import { handleSchemaValidationFailure } from '@src/processors/shared/schema-failure-handler.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; @@ -19,8 +19,10 @@ async function tryFallback( channel: string, notification: T, primaryError: DeliveryResult['error'], - currentProviderId?: string + currentProviderId?: string, + priorAttempts: ProviderAttempt[] = [] ): Promise { + const attempts = [...priorAttempts]; const fallbackIds = PluginRegistry.getFallbackProviderIds(channel); if (!fallbackIds || fallbackIds.length === 0) { logger.debug(`[ProviderRouter] No fallback providers configured for ${channel}`); @@ -41,6 +43,14 @@ async function tryFallback( const fallbackProvider = PluginRegistry.get(fallbackId); if (!fallbackProvider) { logger.warn(`[ProviderRouter] Fallback provider '${fallbackId}' not found in registry, continuing cascade`); + attempts.push({ + provider: fallbackId, + status: 'failed', + error_code: 'PROVIDER_NOT_FOUND', + error_message: `Fallback provider '${fallbackId}' not found in registry`, + retryable: false, + attempted_at: new Date() + }); continue; } @@ -53,6 +63,14 @@ async function tryFallback( message: `Fallback provider '${fallbackId}' is rate limited`, retryable: true }; + attempts.push({ + provider: fallbackId, + status: 'rate_limited', + error_code: 'FALLBACK_RATE_LIMITED', + error_message: `Fallback provider '${fallbackId}' is rate limited`, + retryable: true, + attempted_at: new Date() + }); continue; } @@ -81,21 +99,40 @@ async function tryFallback( validationResult.error, `fallback provider ${fallbackId}` ); + const schemaErrMsg = validationResult.error.issues.map(iss => `${iss.path.join('.')}: ${iss.message}`).join(', '); lastError = { code: 'FALLBACK_SCHEMA_VALIDATION_ERROR', - message: `Fallback provider ${fallbackId} schema validation failed`, + message: `Fallback provider ${fallbackId} schema validation failed: ${schemaErrMsg}`, retryable: false }; + attempts.push({ + provider: fallbackId, + status: 'failed', + error_code: 'FALLBACK_SCHEMA_VALIDATION_ERROR', + error_message: lastError.message, + retryable: false, + attempted_at: new Date() + }); continue; } + const startTime = Date.now(); try { const fallbackResult = await fallbackProvider.send(notification); + const duration_ms = Date.now() - startTime; if (fallbackResult.success) { logger.success(`[${channel}] Notification delivered via fallback provider: ${fallbackId}`); + attempts.push({ + provider: fallbackId, + status: 'delivered', + attempted_at: new Date(), + duration_ms + }); return { ...fallbackResult, + provider: fallbackId, messageId: fallbackResult.messageId || `fallback-${fallbackId}`, + attempts }; } @@ -104,13 +141,32 @@ async function tryFallback( message: `Fallback provider ${fallbackId} send failed without error details`, retryable: false }; + attempts.push({ + provider: fallbackId, + status: 'failed', + error_code: lastError.code, + error_message: lastError.message, + retryable: lastError.retryable, + attempted_at: new Date(), + duration_ms + }); } catch (sendErr) { + const duration_ms = Date.now() - startTime; logger.error(`[${channel}] Error sending via fallback provider ${fallbackId}:`, sendErr); lastError = { code: 'FALLBACK_EXECUTION_ERROR', message: sendErr instanceof Error ? sendErr.message : String(sendErr), retryable: false }; + attempts.push({ + provider: fallbackId, + status: 'failed', + error_code: lastError.code, + error_message: lastError.message, + retryable: false, + attempted_at: new Date(), + duration_ms + }); } } @@ -131,6 +187,7 @@ async function tryFallback( message: `All providers in cascade failed. Last error: ${lastError?.message || 'Unknown'}`, retryable: false, }, + attempts }; } @@ -194,19 +251,52 @@ export async function sendWithFallback( channel: string, notification: T ): Promise { + const attempts: ProviderAttempt[] = []; + // 0. Use explicit provider if specified if (notification.provider) { logger.debug(`[ProviderRouter] Using explicit provider: ${notification.provider}`); - const result = await sendToProvider(notification.provider, notification); + const providerId = notification.provider; + const startTime = Date.now(); + const result = await sendToProvider(providerId, notification); + const duration_ms = Date.now() - startTime; + + if (result.success) { + attempts.push({ + provider: providerId, + status: 'delivered', + attempted_at: new Date(), + duration_ms + }); + return { + ...result, + provider: providerId, + attempts + }; + } - // If success or retryable error, return as-is - if (result.success || result.error?.retryable) { - return result; + attempts.push({ + provider: providerId, + status: 'failed', + error_code: result.error?.code, + error_message: result.error?.message, + retryable: result.error?.retryable, + attempted_at: new Date(), + duration_ms + }); + + // If retryable error, return as-is without fallback + if (result.error?.retryable) { + return { + ...result, + provider: providerId, + attempts + }; } // Non-retryable failure - try fallback provider - const fallbackResult = await tryFallback(channel, notification, result.error, notification.provider); - return fallbackResult ?? result; + const fallbackResult = await tryFallback(channel, notification, result.error, providerId, attempts); + return fallbackResult ?? { ...result, provider: providerId, attempts }; } const defaultProvider = PluginRegistry.getDefaultProvider(channel); @@ -219,21 +309,48 @@ export async function sendWithFallback( message: `No provider configured for channel: ${channel}`, retryable: false, }, + attempts }; } - const defaultProviderId = PluginRegistry.getDefaultProviderId(channel); + const defaultProviderId = PluginRegistry.getDefaultProviderId(channel) || 'default'; // Try default provider + const startTime = Date.now(); const result = await defaultProvider.send(notification); + const duration_ms = Date.now() - startTime; if (result.success) { - return result; + attempts.push({ + provider: defaultProviderId, + status: 'delivered', + attempted_at: new Date(), + duration_ms + }); + return { + ...result, + provider: defaultProviderId, + attempts + }; } + attempts.push({ + provider: defaultProviderId, + status: 'failed', + error_code: result.error?.code, + error_message: result.error?.message, + retryable: result.error?.retryable, + attempted_at: new Date(), + duration_ms + }); + // If error is retryable, don't fallback - let SimpleNS retry with same provider if (result.error?.retryable) { - return result; + return { + ...result, + provider: defaultProviderId, + attempts + }; } // Try fallback provider @@ -241,9 +358,10 @@ export async function sendWithFallback( channel, notification, result.error, - defaultProviderId + defaultProviderId, + attempts ); - return fallbackResult ?? result; + return fallbackResult ?? { ...result, provider: defaultProviderId, attempts }; } /** diff --git a/src/processors/unified/unified.consumer.ts b/src/processors/unified/unified.consumer.ts index 1b006ce..096998e 100644 --- a/src/processors/unified/unified.consumer.ts +++ b/src/processors/unified/unified.consumer.ts @@ -45,7 +45,7 @@ import status_outbox_model from '@src/database/models/status-outbox.models.js'; // Re-export consumer health types export type { ConsumerHealthState, ConsumerHealthCheckResult, ConsumerHealthDetails }; -// Track active consumers and their health status by channel +// State tracking for graceful shutdown & health const consumers: Map = new Map(); const consumingState: Map = new Map(); const consumerHealthState: Map = new Map(); @@ -78,6 +78,8 @@ const publishSuccessStatus = async ( request_id: notification.request_id, client_id: notification.client_id, channel: channel, + provider: notification.provider, + provider_history: notification.provider_history, status: NOTIFICATION_STATUS_SF.delivered, message: messageId ? `Delivered via ${messageId}` : 'Notification sent successfully', retry_count: notification.retry_count, @@ -102,6 +104,8 @@ const publishFailureStatus = async ( request_id: notification.request_id, client_id: notification.client_id, channel: channel, + provider: notification.provider, + provider_history: notification.provider_history, status: NOTIFICATION_STATUS_SF.failed, message: errorMessage, retry_count: notification.retry_count, @@ -272,6 +276,9 @@ const processMessage = async ( } const notification = validationResult.data as BaseNotification; + if (!notification.provider_history) { + notification.provider_history = []; + } const notificationId = notification.notification_id.toString(); logger.info(`[${channel}] Processing notification: ${notificationId} (retry: ${notification.retry_count})`); @@ -312,6 +319,15 @@ const processMessage = async ( break; } else { lastRateLimitResult = rateLimitResult; + const retryDelay = rateLimitResult.retryAfterMs ?? env.RATE_LIMIT_RETRY_DELAY_MS; + notification.provider_history.push({ + provider: candidateId, + status: 'rate_limited', + error_code: 'RATE_LIMITED', + error_message: `Provider '${candidateId}' rate limit exceeded (retry after ${retryDelay}ms)`, + retryable: true, + attempted_at: new Date() + }); logger.warn(`[${channel}] Provider '${candidateId}' rate limited (${i + 1}/${cascade.length}): ${notificationId}`); } } @@ -362,6 +378,16 @@ const processMessage = async ( // 5. Send via plugin router (with auto-fallback) const result: DeliveryResult = await sendWithFallback(channel, notification); + // Merge router dispatch attempts into provider_history + if (result.attempts && result.attempts.length > 0) { + notification.provider_history.push(...result.attempts); + } + + // Update notification provider if result specifies the delivering/failing provider + if (result.provider && result.provider !== notification.provider) { + notification.provider = result.provider; + } + if (result.success) { // 6a. Success try { @@ -390,6 +416,8 @@ const processMessage = async ( await status_outbox_model.create({ notification_id: notificationId, status: NOTIFICATION_STATUS_SF.delivered, + provider: notification.provider, + provider_history: notification.provider_history, processed: false }); logger.info(`[${channel}] Created status_outbox fallback for: ${notificationId}`); diff --git a/src/types/schemas.ts b/src/types/schemas.ts index 7353cf5..ad045c8 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -40,12 +40,24 @@ const hasNonEmptyContent = ( content?: Record>, ) => !!content && Object.keys(content).length > 0; +export const providerAttemptSchema = z.object({ + provider: z.string(), + status: z.enum(['delivered', 'failed', 'rate_limited']), + error_code: z.string().optional(), + error_message: z.string().optional(), + retryable: z.boolean().optional(), + attempted_at: z.coerce.date(), + duration_ms: z.number().optional(), + response: z.unknown().optional(), +}); + const notificationPayloadFields = { request_id: UUIDV4Schema, client_id: UUIDV4Schema, client_name: z.string().optional(), channel: z.string(), provider: z.string().optional(), + provider_history: z.array(providerAttemptSchema).optional(), recipient: z.record(z.string(), z.unknown()), content: z.record(z.string(), z.unknown()), variables: variablesSchema.optional(), @@ -112,6 +124,8 @@ export const notificationStatusTopicSchema = z.object({ request_id: UUIDV4Schema, client_id: UUIDV4Schema, channel: z.string(), + provider: z.string().optional(), + provider_history: z.array(providerAttemptSchema).optional(), status: z.enum(NOTIFICATION_STATUS_SF), message: z.string(), retry_count: z.number().int().min(0), @@ -178,6 +192,8 @@ export const statusOutboxSchema = z.object({ _id: objectIdSchema, notification_id: objectIdSchema, status: z.enum(NOTIFICATION_STATUS_SF), + provider: z.string().optional(), + provider_history: z.array(providerAttemptSchema).optional(), processed: z.boolean().default(false), claimed_by: z.string().nullable().optional(), claimed_at: z.coerce.date().nullable().optional(), diff --git a/src/types/types.ts b/src/types/types.ts index ed75ddd..c96c5f8 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -68,12 +68,14 @@ export type RefillInterval = 'second' | 'minute' | 'hour' | 'day'; export interface DeliveryResult { success: boolean; messageId?: string; + provider?: string; providerResponse?: unknown; error?: { code: string; message: string; retryable: boolean; }; + attempts?: ProviderAttempt[]; } export interface ProviderConfig { @@ -110,6 +112,7 @@ import { baseNotificationRequestSchema, baseBatchNotificationRequestSchema, baseNotificationSchema, + providerAttemptSchema, // Admin channel schemas adminChannelSchema, systemConfigSchema, @@ -161,6 +164,8 @@ export type status_outbox = z.infer; export type base_notification = z.infer; +export type ProviderAttempt = z.infer; + export type BaseNotification = Omit & { notification_id: string; }; diff --git a/src/workers/consumers/status.consumer.ts b/src/workers/consumers/status.consumer.ts index e408199..f82842f 100644 --- a/src/workers/consumers/status.consumer.ts +++ b/src/workers/consumers/status.consumer.ts @@ -113,6 +113,14 @@ const processStatusMessage = async ({ partition, message }: EachMessagePayload): updated_at: new Date() }; + if (data.provider) { + updateData.provider = data.provider; + } + + if (data.provider_history && data.provider_history.length > 0) { + updateData.provider_history = data.provider_history; + } + // Store error message if failed if (newStatus === NOTIFICATION_STATUS.failed) { updateData.error_message = data.message; diff --git a/src/workers/producers/background.producer.ts b/src/workers/producers/background.producer.ts index 5bbdf5f..0ca1864 100644 --- a/src/workers/producers/background.producer.ts +++ b/src/workers/producers/background.producer.ts @@ -182,6 +182,8 @@ export const sendStatusOutboxEvents = async (statusEntries: status_outbox[]): Pr request_id: notification.request_id, client_id: notification.client_id, channel: notification.channel, + provider: entry.provider || notification.provider, + provider_history: entry.provider_history || notification.provider_history, status: entry.status as NOTIFICATION_STATUS_SF, message: entry.status === 'delivered' ? 'Recovered by recovery service - ghost delivery' diff --git a/tests/integration/workers/consumers/status.consumer.test.ts b/tests/integration/workers/consumers/status.consumer.test.ts index 1ed5b87..3d5360f 100644 --- a/tests/integration/workers/consumers/status.consumer.test.ts +++ b/tests/integration/workers/consumers/status.consumer.test.ts @@ -281,6 +281,46 @@ describe('Status Consumer', () => { ); }); + it('should update provider and provider_history in MongoDB when provided', async () => { + const history = [ + { + provider: 'primary', + status: 'rate_limited', + error_code: 'RATE_LIMITED', + error_message: 'Rate limit exceeded', + retryable: true, + attempted_at: new Date().toISOString(), + }, + { + provider: 'fallback', + status: 'delivered', + attempted_at: new Date().toISOString(), + duration_ms: 120, + } + ]; + const statusData = createStatusMessage({ + status: 'delivered', + provider: 'fallback', + provider_history: history, + }); + const kafkaMessage = createKafkaMessage(statusData); + + await capturedMessageHandler!(kafkaMessage); + + expect(mockFindByIdAndUpdate).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + status: 'delivered', + provider: 'fallback', + provider_history: expect.arrayContaining([ + expect.objectContaining({ provider: 'primary', status: 'rate_limited' }), + expect.objectContaining({ provider: 'fallback', status: 'delivered' }), + ]), + }), + expect.anything() + ); + }); + it('should send webhook callback when webhook_url is provided', async () => { const statusData = createStatusMessage({ webhook_url: 'https://example.com/webhook', diff --git a/tests/unit/plugins/router.test.ts b/tests/unit/plugins/router.test.ts index 0c21ec9..e988ff8 100644 --- a/tests/unit/plugins/router.test.ts +++ b/tests/unit/plugins/router.test.ts @@ -165,6 +165,41 @@ describe('Provider Router', () => { expect(fallback.send).toHaveBeenCalled(); }); + it('should record execution attempts with timing, provider, and error details', async () => { + const primary = createMockProvider({ + name: 'primary', + channel: 'email', + sendResult: { + success: false, + error: { code: 'AUTH_FAILED', message: 'Invalid API key', retryable: false } + } + }); + const fallback = createMockProvider({ name: 'fallback', channel: 'email' }); + + PluginRegistry.register(primary as any, 'primary', 2); + PluginRegistry.register(fallback as any, 'fallback', 1); + PluginRegistry.setChannelConfig('email', { default: 'primary', fallback: 'fallback' }); + + const result = await sendWithFallback('email', createNotification('email')); + + expect(result.success).toBe(true); + expect(result.provider).toBe('fallback'); + expect(result.attempts).toHaveLength(2); + expect(result.attempts?.[0]).toMatchObject({ + provider: 'primary', + status: 'failed', + error_code: 'AUTH_FAILED', + error_message: 'Invalid API key', + retryable: false, + }); + expect(result.attempts?.[0].duration_ms).toBeDefined(); + expect(result.attempts?.[1]).toMatchObject({ + provider: 'fallback', + status: 'delivered', + }); + expect(result.attempts?.[1].duration_ms).toBeDefined(); + }); + it('should NOT try fallback on retryable error', async () => { const primary = createMockProvider({ name: 'primary', diff --git a/tests/unit/processors/unified.consumer.test.ts b/tests/unit/processors/unified.consumer.test.ts index 339e995..9f9c3e5 100644 --- a/tests/unit/processors/unified.consumer.test.ts +++ b/tests/unit/processors/unified.consumer.test.ts @@ -351,6 +351,18 @@ describe('processMessage via consumer run loop', () => { expect.objectContaining({ provider: 'fallback' }) ); expect(setDeliveredMock).toHaveBeenCalledWith('notif-123', 0); + expect(publishStatusMock).toHaveBeenCalledWith(expect.objectContaining({ + notification_id: 'notif-123', + status: 'delivered', + provider: 'fallback', + provider_history: expect.arrayContaining([ + expect.objectContaining({ + provider: 'primary', + status: 'rate_limited', + error_code: 'RATE_LIMITED' + }) + ]) + })); expect(commitOffsetsMock).toHaveBeenCalledTimes(1); }); From 11a70e87024dbb0b55a89c1ac876321632b76d29 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Mon, 21 Sep 2026 18:23:16 +0530 Subject: [PATCH 26/46] feat: add provider rate limit management endpoints and logic - Implemented API endpoints for retrieving and resetting provider rate limits. - Added types and schemas for provider rate limit configurations and statuses. - Enhanced rate limiter logic to support batch querying and telemetry for multiple providers. - Updated Redis key management to include new rate limit metrics. - Added unit tests for the new endpoints and rate limiter functionalities. --- .../components/plugins/providers-tab.tsx | 251 +++++++++++++++- dashboard/lib/api-client.ts | 8 + dashboard/lib/types.ts | 41 +++ src/api/controllers/providers.controller.ts | 69 +++++ src/api/routes/providers.routes.ts | 6 + src/plugins/loader/registry.ts | 14 + .../services/provider-manager.service.ts | 4 + src/processors/shared/rate-limiter.ts | 277 ++++++++++++++++-- src/types/schemas.ts | 55 ++++ src/types/types.ts | 16 + tests/unit/api/providers.controller.test.ts | 164 +++++++++++ tests/unit/shared/rate-limiter.test.ts | 158 +++++++--- 12 files changed, 991 insertions(+), 72 deletions(-) create mode 100644 tests/unit/api/providers.controller.test.ts diff --git a/dashboard/components/plugins/providers-tab.tsx b/dashboard/components/plugins/providers-tab.tsx index 7f5f859..3725da0 100644 --- a/dashboard/components/plugins/providers-tab.tsx +++ b/dashboard/components/plugins/providers-tab.tsx @@ -1,15 +1,20 @@ "use client"; -import { useState } from "react"; +import { useState, useMemo } from "react"; import useSWR from "swr"; import { providerService, pluginService } from "@/lib/api-client"; -import type { ProviderDto, InstalledPlugin } from "@/lib/types"; +import type { ProviderDto, InstalledPlugin, ProviderRateLimitStatus, ProviderRateLimitListResponse } from "@/lib/types"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; import { Badge } from "@/components/ui/badge"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Switch } from "@/components/ui/switch"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; import { Dialog, DialogContent, @@ -49,8 +54,12 @@ import { Server, Eye, EyeOff, + Gauge, + RotateCcw, + AlertTriangle, } from "lucide-react"; import { toast } from "sonner"; +import { cn } from "@/lib/utils"; type AdditionalOption = { id: string; @@ -66,6 +75,188 @@ const createAdditionalOption = (key = "", value = ""): AdditionalOption => ({ value, }); +function ProviderRateLimitRing({ + rateLimit, + onReset, + isResetting, +}: { + rateLimit?: ProviderRateLimitStatus; + onReset: () => void; + isResetting?: boolean; +}) { + const [open, setOpen] = useState(false); + + if (!rateLimit) { + return null; + } + + const { config, status } = rateLimit; + const maxTokens = config.max_tokens; + const remaining = status.remaining_tokens; + const used = status.used_tokens; + const isExhausted = status.is_exhausted; + + const remainingPct = maxTokens > 0 ? Math.min(100, Math.max(0, Math.round((remaining / maxTokens) * 100))) : 0; + const usedPct = 100 - remainingPct; + + let strokeColor = "text-emerald-500"; + let statusBadgeClass = "bg-emerald-500/15 text-emerald-600 dark:text-emerald-400 border-emerald-500/30"; + let statusText = "Healthy"; + + if (isExhausted) { + strokeColor = "text-rose-500"; + statusBadgeClass = "bg-rose-500/15 text-rose-600 dark:text-rose-400 border-rose-500/30"; + statusText = "Exhausted"; + } else if (remainingPct <= 30) { + strokeColor = "text-amber-500"; + statusBadgeClass = "bg-amber-500/15 text-amber-600 dark:text-amber-400 border-amber-500/30"; + statusText = "Low Capacity"; + } + + const size = 32; + const strokeWidth = 3; + const center = size / 2; + const radius = center - strokeWidth; + const circumference = 2 * Math.PI * radius; + const strokeDashoffset = circumference - (remainingPct / 100) * circumference; + + return ( + + + + + setOpen(true)} + onMouseLeave={() => setOpen(false)} + onClick={(e) => e.stopPropagation()} + > +
+
+ + Rate Limit Details +
+ + {statusText} + +
+ +
+
+ Tokens Remaining: + {remaining} / {maxTokens} +
+
+
+
+
+ Used: {used} ({usedPct}%) + Capacity: {maxTokens} +
+
+ +
+
+ Refill Cadence: + +{config.refill_rate} / {config.refill_interval} +
+ {status.is_exhausted ? ( +
+ + Next token in {(status.resets_in_ms / 1000).toFixed(1)}s +
+ ) : remaining < maxTokens ? ( +
+ Full bucket in: + {(status.full_refill_in_ms / 1000).toFixed(1)}s +
+ ) : ( +
+ Bucket is at full capacity +
+ )} + {status.exhausted_count > 0 && ( +
+ Exhaustion events: + {status.exhausted_count} +
+ )} +
+ +
+ +
+ + + ); +} + export function ProvidersTab() { const { data: providersData, @@ -78,6 +269,41 @@ export function ProvidersTab() { () => pluginService.listInstalled() ); + const { + data: rateLimitsData, + mutate: mutateRateLimits, + } = useSWR( + "/api/providers/rate-limits", + () => providerService.getRateLimits(), + { refreshInterval: 5000, revalidateOnFocus: true } + ); + + const rateLimitsMap = useMemo(() => { + const map = new Map(); + if (rateLimitsData?.providers) { + for (const rl of rateLimitsData.providers) { + map.set(rl.provider_id, rl); + } + } + return map; + }, [rateLimitsData]); + + const [resettingProviderId, setResettingProviderId] = useState(null); + + const handleResetRateLimit = async (id: string) => { + try { + setResettingProviderId(id); + await providerService.resetRateLimit(id); + toast.success(`Rate limit for provider '${id}' reset successfully`); + await mutateRateLimits(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to reset rate limit"; + toast.error(message); + } finally { + setResettingProviderId(null); + } + }; + const [modalOpen, setModalOpen] = useState(false); const [editingProvider, setEditingProvider] = useState(null); @@ -264,6 +490,7 @@ export function ProvidersTab() { setModalOpen(false); resetForm(); mutateProviders(); + mutateRateLimits(); } catch (err: unknown) { const message = err instanceof Error ? err.message : "Failed to save provider"; toast.error(message); @@ -281,6 +508,7 @@ export function ProvidersTab() { setDeleteDialogOpen(false); setProviderToDelete(null); mutateProviders(); + mutateRateLimits(); } catch (err: unknown) { const message = err instanceof Error ? err.message : "Failed to delete provider"; toast.error(message); @@ -294,6 +522,7 @@ export function ProvidersTab() { await providerService.update(provider.id, { enabled: newEnabled }); toast.success(`Provider '${provider.id}' ${newEnabled ? "enabled" : "disabled"}`); mutateProviders(); + mutateRateLimits(); } catch (err: unknown) { const message = err instanceof Error ? err.message : "Failed to update provider status"; toast.error(message); @@ -316,7 +545,10 @@ export function ProvidersTab() {
- - {p.enabled ? "Enabled" : "Disabled"} - +
+ handleResetRateLimit(p.id)} + isResetting={resettingProviderId === p.id} + /> + + {p.enabled ? "Enabled" : "Disabled"} + +
diff --git a/dashboard/lib/api-client.ts b/dashboard/lib/api-client.ts index ad0504d..ddd531d 100644 --- a/dashboard/lib/api-client.ts +++ b/dashboard/lib/api-client.ts @@ -27,6 +27,8 @@ import { ChannelRoutingDto, OperationalSettings, AdminAuthStatus, + ProviderRateLimitStatus, + ProviderRateLimitListResponse, } from './types'; export class ApiError extends Error { @@ -194,6 +196,12 @@ export const providerService = { options?: Record; }): Promise<{ success: boolean; message: string }> => apiClient.post('/api/providers/test', payload), + getRateLimits: (): Promise => + apiClient.get('/api/providers/rate-limits'), + getRateLimit: (id: string): Promise<{ rate_limit: ProviderRateLimitStatus }> => + apiClient.get(`/api/providers/${id}/rate-limit`), + resetRateLimit: (id: string): Promise<{ message: string }> => + apiClient.post(`/api/providers/${id}/rate-limit/reset`), }; export const channelRoutingService = { diff --git a/dashboard/lib/types.ts b/dashboard/lib/types.ts index 267e005..4a22295 100644 --- a/dashboard/lib/types.ts +++ b/dashboard/lib/types.ts @@ -361,3 +361,44 @@ export interface AdminAuthStatus { isConfigured: boolean; } +export interface ProviderRateLimitConfigDetails { + max_tokens: number; + refill_rate: number; + refill_interval: "second" | "minute" | "hour" | "day"; + normalized_refill_rate: number; +} + +export interface ProviderRateLimitRealtimeStatus { + remaining_tokens: number; + used_tokens: number; + usage_percentage: number; + is_exhausted: boolean; + resets_in_ms: number; + full_refill_in_ms: number; + last_refill_at: string | null; + last_exhausted_at: string | null; + exhausted_count: number; +} + +export interface ProviderRateLimitStatus { + provider_id: string; + channel: string; + plugin_name: string; + display_name?: string; + enabled: boolean; + config: ProviderRateLimitConfigDetails; + status: ProviderRateLimitRealtimeStatus; +} + +export interface ProviderRateLimitSummary { + total_providers: number; + exhausted_providers: number; + healthy_providers: number; +} + +export interface ProviderRateLimitListResponse { + providers: ProviderRateLimitStatus[]; + summary: ProviderRateLimitSummary; +} + + diff --git a/src/api/controllers/providers.controller.ts b/src/api/controllers/providers.controller.ts index 97228b7..7220143 100644 --- a/src/api/controllers/providers.controller.ts +++ b/src/api/controllers/providers.controller.ts @@ -186,3 +186,72 @@ export const testProvider = async (req: Request, res: Response): Promise = }); } }; + +/** + * GET /api/providers/rate-limits + * Get real-time rate limit status across all configured providers + */ +export const getProvidersRateLimits = async (_req: Request, res: Response): Promise => { + try { + const { getAllProvidersRateLimitStatus } = await import('@src/processors/shared/rate-limiter.js'); + const result = await getAllProvidersRateLimitStatus(); + res.json(result); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error fetching providers rate limits: ${message}`); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to retrieve providers rate limits' + }); + } +}; + +/** + * GET /api/providers/:id/rate-limit + * Get real-time rate limit status for a specific provider + */ +export const getProviderRateLimit = async (req: Request, res: Response): Promise => { + try { + const { getProviderRateLimitStatus } = await import('@src/processors/shared/rate-limiter.js'); + const rateLimitStatus = await getProviderRateLimitStatus(req.params.id); + + if (!rateLimitStatus) { + res.status(404).json({ + error: 'Not Found', + message: `Provider '${req.params.id}' rate limit status not found` + }); + return; + } + + res.json({ rate_limit: rateLimitStatus }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error fetching provider '${req.params.id}' rate limit: ${message}`); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to retrieve provider rate limit' + }); + } +}; + +/** + * POST /api/providers/:id/rate-limit/reset + * Manually reset rate limiter tokens and telemetry for a provider + */ +export const resetProviderRateLimit = async (req: Request, res: Response): Promise => { + try { + const { resetRateLimiter } = await import('@src/processors/shared/rate-limiter.js'); + await resetRateLimiter(req.params.id); + res.json({ + message: `Rate limit for provider '${req.params.id}' has been reset successfully.` + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error resetting provider '${req.params.id}' rate limit: ${message}`); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to reset provider rate limit' + }); + } +}; + diff --git a/src/api/routes/providers.routes.ts b/src/api/routes/providers.routes.ts index 6f6045f..467bb6a 100644 --- a/src/api/routes/providers.routes.ts +++ b/src/api/routes/providers.routes.ts @@ -12,12 +12,18 @@ import { updateProvider, deleteProvider, testProvider, + getProvidersRateLimits, + getProviderRateLimit, + resetProviderRateLimit, } from '../controllers/providers.controller.js'; const router = Router(); router.get('/', listProviders); router.post('/test', testProvider); +router.get('/rate-limits', getProvidersRateLimits); +router.get('/:id/rate-limit', getProviderRateLimit); +router.post('/:id/rate-limit/reset', resetProviderRateLimit); router.get('/:id', getProvider); router.post('/', createProvider); router.put('/:id', updateProvider); diff --git a/src/plugins/loader/registry.ts b/src/plugins/loader/registry.ts index 296793b..245eeef 100644 --- a/src/plugins/loader/registry.ts +++ b/src/plugins/loader/registry.ts @@ -284,6 +284,20 @@ class PluginRegistryClass { return this.providers.has(id); } + /** + * Get all registered provider entries + */ + getAllRegistered(): RegisteredProvider[] { + return Array.from(this.providers.values()); + } + + /** + * Get registered provider entry by ID + */ + getRegistered(id: string): RegisteredProvider | undefined { + return this.providers.get(id); + } + /** * Check if a channel has providers */ diff --git a/src/plugins/services/provider-manager.service.ts b/src/plugins/services/provider-manager.service.ts index 18210e0..d40da92 100644 --- a/src/plugins/services/provider-manager.service.ts +++ b/src/plugins/services/provider-manager.service.ts @@ -25,6 +25,7 @@ import type { encrypted_credentials, } from '@src/types/types.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; +import { resetRateLimiter } from '@src/processors/shared/rate-limiter.js'; export type { ProviderResponseDto } from '@src/types/types.js'; @@ -255,6 +256,9 @@ export class ProviderManagerService { await Provider.deleteOne({ id }); ProviderManagerService.invalidateCredentialCache(id); PluginRegistry.unregister(id); + await resetRateLimiter(id).catch((err) => { + logger.warn(`Failed to clean up rate limiter keys for deleted provider '${id}':`, err); + }); await PluginSyncService.publish('PROVIDER_DELETED', { provider_id: id, diff --git a/src/processors/shared/rate-limiter.ts b/src/processors/shared/rate-limiter.ts index 92a0665..d6a0106 100644 --- a/src/processors/shared/rate-limiter.ts +++ b/src/processors/shared/rate-limiter.ts @@ -8,13 +8,16 @@ import { getRedisClient } from '@src/config/redis.config.js'; import { env } from '@src/config/env.config.js'; import { getRateLimitConfig as getPluginRateLimitConfig } from '@src/plugins/index.js'; +import { PluginRegistry } from '@src/plugins/loader/registry.js'; +import Provider from '@src/database/models/provider.models.js'; import { rateLimiterLogger as logger } from '@src/workers/utils/logger.js'; -import type { RateLimitConfig, RefillInterval, RateLimitResult } from '@src/types/types.js'; - -// Redis key prefixes -const TOKENS_KEY_PREFIX = 'ratelimit:tokens'; -const LAST_REFILL_KEY_PREFIX = 'ratelimit:last_refill'; -const QUEUE_POSITION_KEY_PREFIX = 'ratelimit:queue_position'; +import type { + RateLimitConfig, + RefillInterval, + RateLimitResult, + ProviderRateLimitStatus, + ProviderRateLimitListResponse, +} from '@src/types/types.js'; // Conversion divisors to convert rate to per-second const INTERVAL_TO_SECONDS: Record = { @@ -28,14 +31,14 @@ const INTERVAL_TO_SECONDS: Record = { * Normalize refill rate to tokens per second * Converts interval-based rates (e.g., 500/day) to per-second rates */ -function normalizeRefillRate(config: RateLimitConfig): number { +export function normalizeRefillRate(config: RateLimitConfig): number { const interval = config.refillInterval || 'second'; const divisor = INTERVAL_TO_SECONDS[interval] || 1; return config.refillRate / divisor; } // Default rate limit config -const DEFAULT_RATE_LIMIT: RateLimitConfig = { +export const DEFAULT_RATE_LIMIT: RateLimitConfig = { maxTokens: 100, refillRate: 10, refillInterval: 'second', @@ -45,7 +48,7 @@ const DEFAULT_RATE_LIMIT: RateLimitConfig = { * Get rate limit configuration for a provider * Priority: Plugin config > Default */ -const getConfig = (providerId: string): RateLimitConfig => { +export const getConfig = (providerId: string): RateLimitConfig => { // Try plugin registry first const pluginConfig = getPluginRateLimitConfig(providerId); if (pluginConfig) { @@ -57,16 +60,35 @@ const getConfig = (providerId: string): RateLimitConfig => { }; /** - * Build Redis keys for a provider + * Build Redis keys for a provider using Redis Cluster hash tags {providerId} */ -const buildKeys = (providerId: string): { tokensKey: string; lastRefillKey: string; queueKey: string } => { +export const buildKeys = (providerId: string): { + tokensKey: string; + lastRefillKey: string; + queueKey: string; + exhaustedCountKey: string; + lastExhaustedKey: string; +} => { return { - tokensKey: `${TOKENS_KEY_PREFIX}:${providerId}`, - lastRefillKey: `${LAST_REFILL_KEY_PREFIX}:${providerId}`, - queueKey: `${QUEUE_POSITION_KEY_PREFIX}:${providerId}` + tokensKey: `ratelimit:{${providerId}}:tokens`, + lastRefillKey: `ratelimit:{${providerId}}:last_refill`, + queueKey: `ratelimit:{${providerId}}:queue_position`, + exhaustedCountKey: `ratelimit:{${providerId}}:exhausted_count`, + lastExhaustedKey: `ratelimit:{${providerId}}:last_exhausted`, }; }; +// In-memory status response cache to debounce high-frequency dashboard queries +let cachedStatusResponse: { data: ProviderRateLimitListResponse; timestamp: number } | null = null; +const CACHE_TTL_MS = 1000; // 1 second + +/** + * Clear the in-memory rate limit status cache + */ +export const clearRateLimitStatusCache = (): void => { + cachedStatusResponse = null; +}; + /** * Try to consume a token from the bucket * Uses Redis Lua script for atomic operation with fixed retry delay @@ -75,22 +97,27 @@ export const consumeToken = async (providerId: string): Promise const redis = getRedisClient(); const config = getConfig(providerId); const normalizedRate = normalizeRefillRate(config); - const { tokensKey, lastRefillKey } = buildKeys(providerId); + const { tokensKey, lastRefillKey, exhaustedCountKey, lastExhaustedKey } = buildKeys(providerId); const fixedRetryDelay = env.RATE_LIMIT_RETRY_DELAY_MS; // Debug logging logger.debug(`Provider: ${providerId}, Config: maxTokens=${config.maxTokens}, refillRate=${config.refillRate}/${config.refillInterval || 'second'} (normalized: ${normalizedRate.toFixed(6)}/sec), retryDelay=${fixedRetryDelay}ms`); const now = Date.now(); + // Rolling TTL: 7 days or at least 2x the duration needed to refill from 0 to full capacity + const ttlSeconds = Math.max(86400 * 7, Math.ceil((config.maxTokens / Math.max(normalizedRate, 0.0001)) * 2)); - // Lua script for atomic token bucket operation with fixed retry delay + // Lua script for atomic token bucket operation with cluster-safe hash tag keys & telemetry const luaScript = ` local tokens_key = KEYS[1] local last_refill_key = KEYS[2] + local exhausted_count_key = KEYS[3] + local last_exhausted_key = KEYS[4] local max_tokens = tonumber(ARGV[1]) local refill_rate = tonumber(ARGV[2]) local now = tonumber(ARGV[3]) local fixed_retry_delay = tonumber(ARGV[4]) + local ttl_seconds = tonumber(ARGV[5]) -- Get current state local current_tokens = tonumber(redis.call('GET', tokens_key)) or max_tokens @@ -104,23 +131,29 @@ export const consumeToken = async (providerId: string): Promise -- Try to consume a token if new_tokens >= 1 then new_tokens = new_tokens - 1 - redis.call('SET', tokens_key, new_tokens) - redis.call('SET', last_refill_key, now) + redis.call('SET', tokens_key, new_tokens, 'EX', ttl_seconds) + redis.call('SET', last_refill_key, now, 'EX', ttl_seconds) return { 1, new_tokens, 0 } else + redis.call('INCR', exhausted_count_key) + redis.call('EXPIRE', exhausted_count_key, ttl_seconds) + redis.call('SET', last_exhausted_key, now, 'EX', ttl_seconds) return { 0, new_tokens, fixed_retry_delay } end `; const result = await redis.eval( luaScript, - 2, + 4, tokensKey, lastRefillKey, + exhaustedCountKey, + lastExhaustedKey, config.maxTokens.toString(), normalizedRate.toString(), now.toString(), - fixedRetryDelay.toString() + fixedRetryDelay.toString(), + ttlSeconds.toString() ) as [number, number, number]; const [allowed, remainingTokens, retryAfterMs] = result; @@ -146,7 +179,7 @@ export const getTokenCount = async (providerId: string): Promise => { const [tokensStr, lastRefillStr] = await redis.mget(tokensKey, lastRefillKey); const currentTokens = tokensStr ? parseFloat(tokensStr) : config.maxTokens; - const lastRefill = lastRefillStr ? parseInt(lastRefillStr) : now; + const lastRefill = lastRefillStr ? parseInt(lastRefillStr, 10) : now; const elapsedSeconds = (now - lastRefill) / 1000; const tokensToAdd = elapsedSeconds * normalizedRate; @@ -154,12 +187,208 @@ export const getTokenCount = async (providerId: string): Promise => { return Math.min(currentTokens + tokensToAdd, config.maxTokens); }; +interface ProviderBasicMeta { + id: string; + channel: string; + plugin_name: string; + display_name?: string; + enabled: boolean; +} + /** - * Reset rate limiter for a provider (for testing) + * Fetch all provider rate limit statuses at scale using batched Redis MGET + */ +export const getAllProvidersRateLimitStatus = async (): Promise => { + const now = Date.now(); + + // Check in-memory cache to debounce frequent dashboard requests + if (cachedStatusResponse && (now - cachedStatusResponse.timestamp) < CACHE_TTL_MS) { + return cachedStatusResponse.data; + } + + const providerMap = new Map(); + + // 1. Gather providers from Database (primary) + try { + const dbProviders = await Provider.find().sort({ channel: 1, 'options.priority': -1 }).lean(); + for (const p of dbProviders) { + providerMap.set(p.id, { + id: p.id, + channel: p.channel, + plugin_name: p.plugin_name, + display_name: p.id, + enabled: p.enabled !== false, + }); + } + } catch { + // If MongoDB query fails or is not connected, continue to registry fallback + } + + // 2. Supplement/fallback from PluginRegistry + try { + const registered = PluginRegistry.getAllRegistered(); + for (const r of registered) { + if (!providerMap.has(r.id)) { + providerMap.set(r.id, { + id: r.id, + channel: r.provider.manifest.channel, + plugin_name: r.provider.manifest.name, + display_name: r.provider.manifest.displayName || r.id, + enabled: true, + }); + } else { + const existing = providerMap.get(r.id)!; + if (r.provider.manifest.displayName) { + existing.display_name = r.provider.manifest.displayName; + } + } + } + } catch { + // Registry check safe fallback + } + + const providersList = Array.from(providerMap.values()); + + if (providersList.length === 0) { + const emptyResponse: ProviderRateLimitListResponse = { + providers: [], + summary: { + total_providers: 0, + exhausted_providers: 0, + healthy_providers: 0, + }, + }; + cachedStatusResponse = { data: emptyResponse, timestamp: now }; + return emptyResponse; + } + + // 3. Build batched Redis keys for single MGET round-trip + const redis = getRedisClient(); + const allKeys: string[] = []; + + for (const p of providersList) { + const { tokensKey, lastRefillKey, exhaustedCountKey, lastExhaustedKey } = buildKeys(p.id); + allKeys.push(tokensKey, lastRefillKey, exhaustedCountKey, lastExhaustedKey); + } + + let mgetResults: (string | null)[]; + try { + mgetResults = await redis.mget(...allKeys); + } catch (redisErr) { + logger.error('Failed to batch query Redis for provider rate limits:', redisErr); + mgetResults = new Array(allKeys.length).fill(null); + } + + // 4. Derive metrics in application memory (O(N) with no write contention) + const providersStatus: ProviderRateLimitStatus[] = []; + let exhaustedCount = 0; + + for (let i = 0; i < providersList.length; i++) { + const p = providersList[i]; + const config = getConfig(p.id); + const normalizedRate = normalizeRefillRate(config); + const maxTokens = config.maxTokens; + + const tokensStr = mgetResults[i * 4]; + const lastRefillStr = mgetResults[i * 4 + 1]; + const exhaustedCountStr = mgetResults[i * 4 + 2]; + const lastExhaustedStr = mgetResults[i * 4 + 3]; + + const currentTokens = tokensStr !== null && tokensStr !== undefined ? parseFloat(tokensStr) : maxTokens; + const lastRefill = lastRefillStr ? parseInt(lastRefillStr, 10) : now; + const elapsedSeconds = Math.max(0, (now - lastRefill) / 1000); + const tokensToAdd = elapsedSeconds * normalizedRate; + const rawRemaining = Math.min(maxTokens, currentTokens + tokensToAdd); + + const remainingTokens = Math.min(maxTokens, Math.max(0, Math.round(rawRemaining * 10) / 10)); + const usedTokens = Math.max(0, Math.round((maxTokens - remainingTokens) * 10) / 10); + const usagePercentage = maxTokens > 0 ? Math.min(100, Math.max(0, Math.round((usedTokens / maxTokens) * 100))) : 0; + const isExhausted = remainingTokens < 1; + + if (isExhausted) { + exhaustedCount++; + } + + let resetsInMs = 0; + if (isExhausted && normalizedRate > 0) { + const tokensNeeded = 1 - remainingTokens; + resetsInMs = Math.ceil((tokensNeeded / normalizedRate) * 1000); + } + + let fullRefillInMs = 0; + if (remainingTokens < maxTokens && normalizedRate > 0) { + const tokensNeeded = maxTokens - remainingTokens; + fullRefillInMs = Math.ceil((tokensNeeded / normalizedRate) * 1000); + } + + const lastRefillAt = lastRefillStr ? new Date(parseInt(lastRefillStr, 10)) : null; + const lastExhaustedAt = lastExhaustedStr ? new Date(parseInt(lastExhaustedStr, 10)) : null; + const providerExhaustedCount = exhaustedCountStr ? parseInt(exhaustedCountStr, 10) : 0; + + providersStatus.push({ + provider_id: p.id, + channel: p.channel, + plugin_name: p.plugin_name, + display_name: p.display_name, + enabled: p.enabled, + config: { + max_tokens: maxTokens, + refill_rate: config.refillRate, + refill_interval: config.refillInterval || 'second', + normalized_refill_rate: normalizedRate, + }, + status: { + remaining_tokens: remainingTokens, + used_tokens: usedTokens, + usage_percentage: usagePercentage, + is_exhausted: isExhausted, + resets_in_ms: resetsInMs, + full_refill_in_ms: fullRefillInMs, + last_refill_at: lastRefillAt, + last_exhausted_at: lastExhaustedAt, + exhausted_count: providerExhaustedCount, + }, + }); + } + + const response: ProviderRateLimitListResponse = { + providers: providersStatus, + summary: { + total_providers: providersStatus.length, + exhausted_providers: exhaustedCount, + healthy_providers: providersStatus.length - exhaustedCount, + }, + }; + + cachedStatusResponse = { data: response, timestamp: now }; + return response; +}; + +/** + * Get rate limit status for a single provider + */ +export const getProviderRateLimitStatus = async (providerId: string): Promise => { + const all = await getAllProvidersRateLimitStatus(); + return all.providers.find((p) => p.provider_id === providerId) || null; +}; + +/** + * Reset rate limiter for a provider (refills bucket to capacity and wipes telemetry) */ export const resetRateLimiter = async (providerId: string): Promise => { const redis = getRedisClient(); - const { tokensKey, lastRefillKey, queueKey } = buildKeys(providerId); + const { tokensKey, lastRefillKey, queueKey, exhaustedCountKey, lastExhaustedKey } = buildKeys(providerId); + + await redis.del( + tokensKey, + lastRefillKey, + queueKey, + exhaustedCountKey, + lastExhaustedKey, + `ratelimit:tokens:${providerId}`, + `ratelimit:last_refill:${providerId}`, + `ratelimit:queue_position:${providerId}` + ); - await redis.del(tokensKey, lastRefillKey, queueKey); + clearRateLimitStatusCache(); }; diff --git a/src/types/schemas.ts b/src/types/schemas.ts index ad045c8..8d5dcc1 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -625,3 +625,58 @@ export const installPluginPayloadSchema = z.object({ }).optional(), }); +/** + * Provider rate limit configuration schema + */ +export const providerRateLimitConfigDetailsSchema = z.object({ + max_tokens: z.number(), + refill_rate: z.number(), + refill_interval: z.enum(['second', 'minute', 'hour', 'day']), + normalized_refill_rate: z.number(), +}); + +/** + * Provider rate limit real-time status schema + */ +export const providerRateLimitRealtimeStatusSchema = z.object({ + remaining_tokens: z.number(), + used_tokens: z.number(), + usage_percentage: z.number(), + is_exhausted: z.boolean(), + resets_in_ms: z.number(), + full_refill_in_ms: z.number(), + last_refill_at: z.coerce.date().nullable(), + last_exhausted_at: z.coerce.date().nullable(), + exhausted_count: z.number(), +}); + +/** + * Provider rate limit status item schema + */ +export const providerRateLimitStatusSchema = z.object({ + provider_id: z.string(), + channel: z.string(), + plugin_name: z.string(), + display_name: z.string().optional(), + enabled: z.boolean(), + config: providerRateLimitConfigDetailsSchema, + status: providerRateLimitRealtimeStatusSchema, +}); + +/** + * Provider rate limit summary schema + */ +export const providerRateLimitSummarySchema = z.object({ + total_providers: z.number(), + exhausted_providers: z.number(), + healthy_providers: z.number(), +}); + +/** + * Provider rate limit list response schema + */ +export const providerRateLimitListResponseSchema = z.object({ + providers: z.array(providerRateLimitStatusSchema), + summary: providerRateLimitSummarySchema, +}); + diff --git a/src/types/types.ts b/src/types/types.ts index c96c5f8..7a9b942 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -140,6 +140,11 @@ import { npmAuthConfigSchema, npmAuthStatusSchema, installPluginPayloadSchema, + providerRateLimitConfigDetailsSchema, + providerRateLimitRealtimeStatusSchema, + providerRateLimitStatusSchema, + providerRateLimitSummarySchema, + providerRateLimitListResponseSchema, } from "./schemas.js"; // ============================================================================ @@ -544,3 +549,14 @@ export interface ConsumerHealthCheckResult { healthy: boolean; details: Record; } + +// ============================================================================ +// PROVIDER RATE LIMIT STATUS TYPES +// ============================================================================ + +export type ProviderRateLimitConfigDetails = z.infer; +export type ProviderRateLimitRealtimeStatus = z.infer; +export type ProviderRateLimitStatus = z.infer; +export type ProviderRateLimitSummary = z.infer; +export type ProviderRateLimitListResponse = z.infer; + diff --git a/tests/unit/api/providers.controller.test.ts b/tests/unit/api/providers.controller.test.ts new file mode 100644 index 0000000..5a7887d --- /dev/null +++ b/tests/unit/api/providers.controller.test.ts @@ -0,0 +1,164 @@ +/** + * Unit Tests for Providers Controller Rate Limits Endpoints + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type { Request, Response } from 'express'; + +const mockGetAllProvidersRateLimitStatus = vi.fn(); +const mockGetProviderRateLimitStatus = vi.fn(); +const mockResetRateLimiter = vi.fn(); + +vi.mock('../../../src/processors/shared/rate-limiter.js', () => ({ + getAllProvidersRateLimitStatus: () => mockGetAllProvidersRateLimitStatus(), + getProviderRateLimitStatus: (id: string) => mockGetProviderRateLimitStatus(id), + resetRateLimiter: (id: string) => mockResetRateLimiter(id), +})); + +describe('Providers Controller - Rate Limits Endpoints', () => { + let mockReq: Partial; + let mockRes: Partial; + let jsonMock: ReturnType; + let statusMock: ReturnType; + + beforeEach(() => { + vi.clearAllMocks(); + jsonMock = vi.fn(); + statusMock = vi.fn().mockReturnThis(); + + mockReq = { + params: {}, + }; + mockRes = { + json: jsonMock, + status: statusMock, + }; + }); + + describe('getProvidersRateLimits', () => { + it('should return 200 with all providers rate limit statuses', async () => { + const { getProvidersRateLimits } = await import( + '../../../src/api/controllers/providers.controller.js' + ); + + const mockData = { + providers: [ + { + provider_id: 'gmail-1', + channel: 'email', + plugin_name: 'nodemailer-gmail', + enabled: true, + config: { max_tokens: 100, refill_rate: 10, refill_interval: 'second', normalized_refill_rate: 10 }, + status: { + remaining_tokens: 85, + used_tokens: 15, + usage_percentage: 15, + is_exhausted: false, + resets_in_ms: 0, + full_refill_in_ms: 1500, + last_refill_at: new Date(), + last_exhausted_at: null, + exhausted_count: 0, + }, + }, + ], + summary: { + total_providers: 1, + exhausted_providers: 0, + healthy_providers: 1, + }, + }; + + mockGetAllProvidersRateLimitStatus.mockResolvedValue(mockData); + + await getProvidersRateLimits(mockReq as Request, mockRes as Response); + + expect(jsonMock).toHaveBeenCalledWith(mockData); + }); + + it('should return 500 when retrieval fails', async () => { + const { getProvidersRateLimits } = await import( + '../../../src/api/controllers/providers.controller.js' + ); + + mockGetAllProvidersRateLimitStatus.mockRejectedValue(new Error('Redis connection failed')); + + await getProvidersRateLimits(mockReq as Request, mockRes as Response); + + expect(statusMock).toHaveBeenCalledWith(500); + expect(jsonMock).toHaveBeenCalledWith({ + error: 'Internal server error', + message: 'Failed to retrieve providers rate limits', + }); + }); + }); + + describe('getProviderRateLimit', () => { + it('should return 200 with specific provider rate limit', async () => { + const { getProviderRateLimit } = await import( + '../../../src/api/controllers/providers.controller.js' + ); + + mockReq.params = { id: 'gmail-1' }; + + const mockSingle = { + provider_id: 'gmail-1', + channel: 'email', + plugin_name: 'nodemailer-gmail', + enabled: true, + config: { max_tokens: 100, refill_rate: 10, refill_interval: 'second', normalized_refill_rate: 10 }, + status: { + remaining_tokens: 90, + used_tokens: 10, + usage_percentage: 10, + is_exhausted: false, + resets_in_ms: 0, + full_refill_in_ms: 1000, + last_refill_at: new Date(), + last_exhausted_at: null, + exhausted_count: 0, + }, + }; + + mockGetProviderRateLimitStatus.mockResolvedValue(mockSingle); + + await getProviderRateLimit(mockReq as Request, mockRes as Response); + + expect(jsonMock).toHaveBeenCalledWith({ rate_limit: mockSingle }); + }); + + it('should return 404 if provider rate limit status is not found', async () => { + const { getProviderRateLimit } = await import( + '../../../src/api/controllers/providers.controller.js' + ); + + mockReq.params = { id: 'unknown-provider' }; + mockGetProviderRateLimitStatus.mockResolvedValue(null); + + await getProviderRateLimit(mockReq as Request, mockRes as Response); + + expect(statusMock).toHaveBeenCalledWith(404); + expect(jsonMock).toHaveBeenCalledWith({ + error: 'Not Found', + message: "Provider 'unknown-provider' rate limit status not found", + }); + }); + }); + + describe('resetProviderRateLimit', () => { + it('should call resetRateLimiter and return success message', async () => { + const { resetProviderRateLimit } = await import( + '../../../src/api/controllers/providers.controller.js' + ); + + mockReq.params = { id: 'gmail-1' }; + mockResetRateLimiter.mockResolvedValue(undefined); + + await resetProviderRateLimit(mockReq as Request, mockRes as Response); + + expect(mockResetRateLimiter).toHaveBeenCalledWith('gmail-1'); + expect(jsonMock).toHaveBeenCalledWith({ + message: "Rate limit for provider 'gmail-1' has been reset successfully.", + }); + }); + }); +}); diff --git a/tests/unit/shared/rate-limiter.test.ts b/tests/unit/shared/rate-limiter.test.ts index 6900af4..3ca39d1 100644 --- a/tests/unit/shared/rate-limiter.test.ts +++ b/tests/unit/shared/rate-limiter.test.ts @@ -1,8 +1,6 @@ /** * Unit Tests for Rate Limiter Module * Tests token bucket rate limiting with mocked Redis - * - * Updated for plugin-based architecture - uses dynamic channel strings. */ import { describe, it, expect, vi, beforeEach } from 'vitest'; @@ -29,6 +27,29 @@ vi.mock('../../../src/config/env.config.js', () => ({ }, })); +vi.mock('../../../src/database/models/provider.models.js', () => ({ + default: { + find: vi.fn().mockReturnValue({ + sort: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + id: 'email-provider', + channel: 'email', + plugin_name: 'nodemailer-gmail', + enabled: true, + }, + { + id: 'sms-provider', + channel: 'sms', + plugin_name: 'twilio-sms', + enabled: true, + }, + ]), + }), + }), + }, +})); + describe('Rate Limiter Module', () => { beforeEach(() => { vi.clearAllMocks(); @@ -61,33 +82,23 @@ describe('Rate Limiter Module', () => { expect(result.retryAfterMs).toBe(5000); }); - it('should use email config for email channel', async () => { + it('should use cluster-safe hash tag keys for Redis evaluation', async () => { const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); mockRedisClient.eval.mockResolvedValue([1, 99, 0]); await consumeToken('email'); - // Verify the Lua script was called with email configuration - // eval(script, numKeys=2, key1, key2, arg1=maxTokens, arg2=refillRate, arg3=now, arg4=fixedRetryDelay) expect(mockRedisClient.eval).toHaveBeenCalled(); const callArgs = mockRedisClient.eval.mock.calls[0]; - expect(callArgs[4]).toBe('100'); // maxTokens for email (ARGV[1]) - expect(callArgs[5]).toBe('10'); // refillRate for email (ARGV[2]) - expect(callArgs[7]).toBe('5000'); // fixed retry delay (ARGV[4]) - }); - - it('should use default config for whatsapp channel', async () => { - const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - - mockRedisClient.eval.mockResolvedValue([1, 99, 0]); - - await consumeToken('whatsapp'); - - const callArgs = mockRedisClient.eval.mock.calls[0]; - // Plugin system defaults all channels to same config now - expect(callArgs[4]).toBe('100'); // maxTokens (default) - expect(callArgs[5]).toBe('10'); // refillRate (default) + expect(callArgs[1]).toBe(4); // 4 keys passed + expect(callArgs[2]).toBe('ratelimit:{email}:tokens'); + expect(callArgs[3]).toBe('ratelimit:{email}:last_refill'); + expect(callArgs[4]).toBe('ratelimit:{email}:exhausted_count'); + expect(callArgs[5]).toBe('ratelimit:{email}:last_exhausted'); + expect(callArgs[6]).toBe('100'); // maxTokens + expect(callArgs[7]).toBe('10'); // refillRate + expect(callArgs[9]).toBe('5000'); // retry delay }); }); @@ -95,7 +106,6 @@ describe('Rate Limiter Module', () => { it('should return max tokens when no record exists', async () => { const { getTokenCount } = await import('../../../src/processors/shared/rate-limiter.js'); - // No existing keys mockRedisClient.mget.mockResolvedValue([null, null]); const count = await getTokenCount('email'); @@ -107,7 +117,6 @@ describe('Rate Limiter Module', () => { const { getTokenCount } = await import('../../../src/processors/shared/rate-limiter.js'); const now = Date.now(); - // Current tokens = 50, last refill = now (no time passed) mockRedisClient.mget.mockResolvedValue(['50', now.toString()]); const count = await getTokenCount('email'); @@ -119,7 +128,6 @@ describe('Rate Limiter Module', () => { it('should account for token refill over time', async () => { const { getTokenCount } = await import('../../../src/processors/shared/rate-limiter.js'); - // 1 second ago with 50 tokens, refill rate of 10/sec const oneSecondAgo = Date.now() - 1000; mockRedisClient.mget.mockResolvedValue(['50', oneSecondAgo.toString()]); @@ -130,15 +138,81 @@ describe('Rate Limiter Module', () => { }); }); + describe('getAllProvidersRateLimitStatus', () => { + it('should batch query Redis and return calculated metrics across providers', async () => { + const { getAllProvidersRateLimitStatus, clearRateLimitStatusCache } = await import( + '../../../src/processors/shared/rate-limiter.js' + ); + clearRateLimitStatusCache(); + + const now = Date.now(); + // Provider 1 (email-provider): 70 tokens, refilled now, 0 exhausted + // Provider 2 (sms-provider): 0 tokens, refilled now, 3 exhausted, last exhausted now + mockRedisClient.mget.mockResolvedValue([ + '70', + now.toString(), + '0', + null, + '0', + now.toString(), + '3', + now.toString(), + ]); + + const response = await getAllProvidersRateLimitStatus(); + + expect(response.providers.length).toBe(2); + expect(response.summary.total_providers).toBe(2); + expect(response.summary.exhausted_providers).toBe(1); + expect(response.summary.healthy_providers).toBe(1); + + const p1 = response.providers.find((p) => p.provider_id === 'email-provider'); + expect(p1).toBeDefined(); + expect(p1?.status.remaining_tokens).toBe(70); + expect(p1?.status.used_tokens).toBe(30); + expect(p1?.status.usage_percentage).toBe(30); + expect(p1?.status.is_exhausted).toBe(false); + + const p2 = response.providers.find((p) => p.provider_id === 'sms-provider'); + expect(p2).toBeDefined(); + expect(p2?.status.remaining_tokens).toBe(0); + expect(p2?.status.used_tokens).toBe(100); + expect(p2?.status.usage_percentage).toBe(100); + expect(p2?.status.is_exhausted).toBe(true); + expect(p2?.status.resets_in_ms).toBeGreaterThan(0); + expect(p2?.status.exhausted_count).toBe(3); + }); + + it('should debounce subsequent requests within CACHE_TTL_MS', async () => { + const { getAllProvidersRateLimitStatus, clearRateLimitStatusCache } = await import( + '../../../src/processors/shared/rate-limiter.js' + ); + clearRateLimitStatusCache(); + + mockRedisClient.mget.mockResolvedValue([null, null, null, null, null, null, null, null]); + + await getAllProvidersRateLimitStatus(); + await getAllProvidersRateLimitStatus(); + + // MGET should only have been called once due to in-memory cache + expect(mockRedisClient.mget).toHaveBeenCalledTimes(1); + }); + }); + describe('resetRateLimiter', () => { - it('should delete rate limiter keys from Redis', async () => { + it('should delete both hash-tagged and legacy rate limiter keys from Redis', async () => { const { resetRateLimiter } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.del.mockResolvedValue(3); + mockRedisClient.del.mockResolvedValue(5); await resetRateLimiter('email'); expect(mockRedisClient.del).toHaveBeenCalledWith( + 'ratelimit:{email}:tokens', + 'ratelimit:{email}:last_refill', + 'ratelimit:{email}:queue_position', + 'ratelimit:{email}:exhausted_count', + 'ratelimit:{email}:last_exhausted', 'ratelimit:tokens:email', 'ratelimit:last_refill:email', 'ratelimit:queue_position:email' @@ -156,8 +230,8 @@ describe('Rate Limiter Module', () => { getRateLimitConfig: vi.fn().mockReturnValue({ maxTokens: 60, refillRate: 60, - refillInterval: 'minute' - }) + refillInterval: 'minute', + }), })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); @@ -166,8 +240,8 @@ describe('Rate Limiter Module', () => { await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - expect(callArgs[4]).toBe('60'); // maxTokens - expect(callArgs[5]).toBe('1'); // 60/minute = 1/second + expect(callArgs[6]).toBe('60'); // maxTokens + expect(callArgs[7]).toBe('1'); // 60/minute = 1/second }); it('should normalize refillInterval: hour to per-second rate', async () => { @@ -175,8 +249,8 @@ describe('Rate Limiter Module', () => { getRateLimitConfig: vi.fn().mockReturnValue({ maxTokens: 3600, refillRate: 3600, - refillInterval: 'hour' - }) + refillInterval: 'hour', + }), })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); @@ -185,8 +259,8 @@ describe('Rate Limiter Module', () => { await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - expect(callArgs[4]).toBe('3600'); // maxTokens - expect(callArgs[5]).toBe('1'); // 3600/hour = 1/second + expect(callArgs[6]).toBe('3600'); // maxTokens + expect(callArgs[7]).toBe('1'); // 3600/hour = 1/second }); it('should normalize refillInterval: day to per-second rate', async () => { @@ -194,8 +268,8 @@ describe('Rate Limiter Module', () => { getRateLimitConfig: vi.fn().mockReturnValue({ maxTokens: 500, refillRate: 500, - refillInterval: 'day' - }) + refillInterval: 'day', + }), })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); @@ -204,8 +278,8 @@ describe('Rate Limiter Module', () => { await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - expect(callArgs[4]).toBe('500'); // maxTokens - const normalizedRate = parseFloat(callArgs[5]); + expect(callArgs[6]).toBe('500'); // maxTokens + const normalizedRate = parseFloat(callArgs[7]); expect(normalizedRate).toBeCloseTo(500 / 86400, 5); }); @@ -213,8 +287,8 @@ describe('Rate Limiter Module', () => { vi.doMock('../../../src/plugins/index.js', () => ({ getRateLimitConfig: vi.fn().mockReturnValue({ maxTokens: 100, - refillRate: 10 - }) + refillRate: 10, + }), })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); @@ -223,8 +297,8 @@ describe('Rate Limiter Module', () => { await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - expect(callArgs[4]).toBe('100'); // maxTokens - expect(callArgs[5]).toBe('10'); // 10/second + expect(callArgs[6]).toBe('100'); // maxTokens + expect(callArgs[7]).toBe('10'); // 10/second }); }); }); From 73f8f3c4a9d2a459793fe930ba2f5e38babc8a7a Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Tue, 22 Sep 2026 11:30:29 +0530 Subject: [PATCH 27/46] feat: update onboarding CLI for SimpleNS version compatibility and improve environment variable handling - Bump version to 1.0.13 in package.json - Enhance README with updated CLI commands and version compatibility details - Add tests for version comparison and environment variable loading - Refactor env-config to differentiate between legacy and modern SimpleNS versions - Update docker-compose templates to support new versioning scheme - Modify onboarding process to allow skipping plugin configuration for SimpleNS > 1.3.0 - Improve security notices for auto-generated credentials based on version --- packages/onboard/README.md | 45 +++-- packages/onboard/package-lock.json | 4 +- packages/onboard/package.json | 2 +- .../onboard/src/__tests__/env-config.test.ts | 65 ++++++- packages/onboard/src/__tests__/infra.test.ts | 24 +++ packages/onboard/src/config/constants.ts | 19 +- packages/onboard/src/env-config.ts | 162 ++++++++++++++---- packages/onboard/src/index.ts | 144 +++++++++++++--- packages/onboard/src/infra.ts | 46 ++++- packages/onboard/src/templates.ts | 110 +++++++++++- 10 files changed, 525 insertions(+), 96 deletions(-) diff --git a/packages/onboard/README.md b/packages/onboard/README.md index d23ba98..d14428c 100644 --- a/packages/onboard/README.md +++ b/packages/onboard/README.md @@ -79,22 +79,22 @@ npx @simplens/onboard --infra --dir /path/to/setup ```bash # Complete setup with all options via CLI (no prompts) -npx @simplens/onboard --full --infra mongo kafka redis nginx --env default --base-path /dashboard --plugin @simplens/mock @simplens/nodemailer-gmail --no-output +npx @simplens/onboard --full --infra mongo kafka redis nginx --env default --base-path /dashboard --app-version latest --skip-plugins --no-output ``` This mode: - Requires `--full` flag to enable non-interactive mode - Requires `--env ` to specify environment mode -- **Auto-generates secure credentials** for NS_API_KEY, AUTH_SECRET, and ADMIN_PASSWORD -- **Auto-generates placeholder credentials** for plugins -- **Auto-generates version values** for CORE_VERSION and DASHBOARD_VERSION +- **Auto-generates secure credentials** for `NS_API_KEY` and `AUTH_SECRET` (and `ADMIN_PASSWORD` for legacy SimpleNS <= 1.3.0) +- **Auto-generates placeholder credentials** for plugins if `--plugin` is provided +- **Auto-generates version value** for `VERSION` (default: `latest`) - All other options are optional with sensible defaults - Services are not auto-started (use `docker compose up -d` manually; fallback: `docker-compose up -d`) **⚠️ IMPORTANT**: Auto-generated credentials are **NOT secure for production**. After setup completes, you **must** update the following in your `.env` file: - `NS_API_KEY` - API authentication key - `AUTH_SECRET` - Session secret for dashboard -- `ADMIN_PASSWORD` - Dashboard admin password +- `ADMIN_PASSWORD` - Dashboard admin password (legacy SimpleNS <= 1.3.0 only; for > 1.3.0, the admin account is set up interactively via the Dashboard at `/setup` upon first boot) - Plugin credentials (if any plugins were installed) The CLI will display a security notice with all credentials that need to be updated. @@ -108,9 +108,11 @@ The CLI will display a security notice with all credentials that need to be upda | `--env ` | Environment setup mode: `default` or `interactive` | Prompted | | `--dir ` | Target directory for setup files | Current directory | | `--base-path ` | Dashboard base path (example: `/dashboard`) | Empty (root) | -| `--core-version ` | Override `CORE_VERSION` in generated `.env` (primarily for `--full`) | `latest` | -| `--dashboard-version ` | Override `DASHBOARD_VERSION` in generated `.env` (primarily for `--full`) | `latest` | +| `--app-version ` | Override `VERSION` in generated `.env` and compose (alias: `--version-tag`) | `latest` | +| `--core-version ` | Backward-compatible alias for `--app-version` | `latest` | +| `--dashboard-version ` | Backward-compatible alias for `--app-version` | `latest` | | `--plugin [plugins...]` | Plugins to install (e.g., `@simplens/mock @simplens/nodemailer-gmail`) | Prompted | +| `--skip-plugins` | Skip plugin configuration during onboarding (install via Dashboard later) | `false` | | `--ssl` | Enable optional SSL automation with Certbot | `false` | | `--ssl-domain ` | Public domain for SSL cert (required with `--ssl` in `--full`) | Prompted | | `--ssl-email ` | Email for Let's Encrypt registration (required with `--ssl` in `--full`) | Prompted | @@ -126,6 +128,19 @@ The CLI will display a security notice with all credentials that need to be upda - `loki` - Loki log aggregation (optional) - `grafana` - Grafana observability dashboard (optional) +## SimpleNS Version Compatibility (> 1.3.0 vs <= 1.3.0) + +SimpleNS versions **> 1.3.0** feature a modernized plugin management system and database-driven dynamic configuration. `@simplens/onboard` automatically selects the appropriate compose templates, volume mounts, and `.env` format based on the selected version: + +- **Unified Image Tag (`VERSION`)**: Both core services (`api`, `worker`, `processors`, `recovery`) and `dashboard` share a single `VERSION` variable in `.env` and `docker-compose.yaml` (default: `latest`). +- **SimpleNS > 1.3.0**: + - **No Shared Plugins Volume**: Services have their own internal plugin directory. The shared `plugin-data` volume is completely removed. + - **Optional Plugins Configuration**: Configuring plugins via `simplens.config.yaml` during onboarding is completely optional. You can skip plugin setup during onboarding (`--skip-plugins`) and install plugins dynamically from the Admin Dashboard. If you configure plugins during onboarding, `simplens.config.yaml` is mounted and `SIMPLENS_CONFIG_PATH` is passed to the container. + - **First-Boot Dashboard Admin Setup**: `ADMIN_USERNAME` and `ADMIN_PASSWORD` are not stored in `.env`. On first launch, navigate to the Dashboard to initialize the administrator account via the guided setup screen (`/setup`). + - **Dynamic Operational Settings**: Operational configurations (outbox interval/batch size, idempotency TTL, recovery timeouts, cleanup retention) are stored in MongoDB and dynamically updated live via the Dashboard Settings page. +- **SimpleNS <= 1.3.0**: + - Automatically activates legacy mode with the shared `plugin-data` volume, required `simplens.config.yaml`, `ADMIN_PASSWORD` in `.env`, and operational settings in `.env`. + ## Workflow 1. **Prerequisites Check** @@ -144,15 +159,12 @@ The CLI will display a security notice with all credentials that need to be upda - Load defaults from `.env.example` - Auto-fill infra connection URLs - Ask for `BASE_PATH` first and reuse it throughout setup - - Prompt for critical values (API keys, passwords) + - Prompt for critical values (API keys, session secret) - Generate `.env` file 4. **Plugin Installation** - - Fetch official plugins from registry - - Interactive multi-select - - Generate `simplens.config.yaml` - - Extract and prompt for plugin credentials - - Append credentials to `.env` + - For SimpleNS > 1.3.0: Plugins are optional. Choose to configure plugins now or skip (`--skip-plugins`) and install later via Dashboard. + - For SimpleNS <= 1.3.0: Interactive plugin selection, generate `simplens.config.yaml`, and append credentials to `.env`. 5. **Service Orchestration** - Optionally start infrastructure services @@ -252,8 +264,7 @@ npx @simplens/onboard \ --full \ --infra mongo kafka redis nginx \ --env default \ - --core-version 1.2 \ - --dashboard-version 1.2 \ + --app-version latest \ --base-path /dashboard \ --plugin @simplens/mock @simplens/nodemailer-gmail \ --dir ./my-simplens-setup \ @@ -271,8 +282,8 @@ npx @simplens/onboard \ npx @simplens/onboard \ --full \ --env default \ - --core-version 1.2 \ - --dashboard-version 1.2 \ + --app-version latest \ + --skip-plugins \ --dir /app/simplens # Then start services in CI: diff --git a/packages/onboard/package-lock.json b/packages/onboard/package-lock.json index dac26a2..34dd339 100644 --- a/packages/onboard/package-lock.json +++ b/packages/onboard/package-lock.json @@ -1,12 +1,12 @@ { "name": "@simplens/onboard", - "version": "1.0.12", + "version": "1.0.13", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@simplens/onboard", - "version": "1.0.12", + "version": "1.0.13", "license": "ISC", "dependencies": { "@clack/prompts": "^1.0.0", diff --git a/packages/onboard/package.json b/packages/onboard/package.json index ed61298..cd7e382 100644 --- a/packages/onboard/package.json +++ b/packages/onboard/package.json @@ -1,6 +1,6 @@ { "name": "@simplens/onboard", - "version": "1.0.12", + "version": "1.0.13", "type": "module", "main": "dist/index.js", "engines": { diff --git a/packages/onboard/src/__tests__/env-config.test.ts b/packages/onboard/src/__tests__/env-config.test.ts index 3d67beb..6c6bf07 100644 --- a/packages/onboard/src/__tests__/env-config.test.ts +++ b/packages/onboard/src/__tests__/env-config.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { normalizeBasePath, validateBasePath } from '../env-config.js'; +import { normalizeBasePath, validateBasePath, isVersionGreaterThan130, loadEnvExample } from '../env-config.js'; describe('env-config base path helpers', () => { it('accepts empty base path for root', () => { @@ -26,3 +26,66 @@ describe('env-config base path helpers', () => { expect(normalizeBasePath(' /dashboard ')).toBe('/dashboard'); }); }); + +describe('isVersionGreaterThan130 version comparison', () => { + it('treats latest, master, and empty as > 1.3.0', () => { + expect(isVersionGreaterThan130('latest')).toBe(true); + expect(isVersionGreaterThan130('master')).toBe(true); + expect(isVersionGreaterThan130(undefined)).toBe(true); + expect(isVersionGreaterThan130('')).toBe(true); + }); + + it('identifies versions <= 1.3.0 as legacy', () => { + expect(isVersionGreaterThan130('1.3.0')).toBe(false); + expect(isVersionGreaterThan130('v1.3.0')).toBe(false); + expect(isVersionGreaterThan130('1.2.9')).toBe(false); + expect(isVersionGreaterThan130('1.0.0')).toBe(false); + expect(isVersionGreaterThan130('0.9.0')).toBe(false); + }); + + it('identifies versions > 1.3.0 as new', () => { + expect(isVersionGreaterThan130('1.3.1')).toBe(true); + expect(isVersionGreaterThan130('v1.3.1')).toBe(true); + expect(isVersionGreaterThan130('1.4.0')).toBe(true); + expect(isVersionGreaterThan130('2.0.0')).toBe(true); + }); +}); + +describe('loadEnvExample templates', () => { + it('returns essential variables matching ./env.example for new versions (> 1.3.0)', async () => { + const envVars = await loadEnvExample(false); + const keys = envVars.map(v => v.key); + + expect(keys).toContain('NS_API_KEY'); + expect(keys).toContain('MONGO_URI'); + expect(keys).toContain('BROKERS'); + expect(keys).toContain('REDIS_URL'); + expect(keys).toContain('AUTH_SECRET'); + expect(keys).toContain('VERSION'); + expect(keys).not.toContain('CORE_VERSION'); + expect(keys).not.toContain('DASHBOARD_VERSION'); + + // Must NOT contain admin credentials or operational settings + expect(keys).not.toContain('ADMIN_USERNAME'); + expect(keys).not.toContain('ADMIN_PASSWORD'); + expect(keys).not.toContain('OUTBOX_POLL_INTERVAL_MS'); + expect(keys).not.toContain('OUTBOX_BATCH_SIZE'); + expect(keys).not.toContain('RECOVERY_BATCH_SIZE'); + expect(keys).not.toContain('IDEMPOTENCY_TTL_SECONDS'); + }); + + it('returns legacy variables including admin credentials and outbox settings for legacy (<= 1.3.0)', async () => { + const envVars = await loadEnvExample(true); + const keys = envVars.map(v => v.key); + + expect(keys).toContain('VERSION'); + expect(keys).not.toContain('CORE_VERSION'); + expect(keys).not.toContain('DASHBOARD_VERSION'); + expect(keys).toContain('ADMIN_USERNAME'); + expect(keys).toContain('ADMIN_PASSWORD'); + expect(keys).toContain('OUTBOX_POLL_INTERVAL_MS'); + expect(keys).toContain('OUTBOX_BATCH_SIZE'); + expect(keys).toContain('RECOVERY_BATCH_SIZE'); + expect(keys).toContain('SIMPLENS_CONFIG_PATH'); + }); +}); diff --git a/packages/onboard/src/__tests__/infra.test.ts b/packages/onboard/src/__tests__/infra.test.ts index 74015f7..ea92669 100644 --- a/packages/onboard/src/__tests__/infra.test.ts +++ b/packages/onboard/src/__tests__/infra.test.ts @@ -27,4 +27,28 @@ describe('infra app compose generation', () => { expect(compose).toContain('certbot-etc:'); expect(compose).toContain('certbot-www:'); }); + + it('generates legacy compose with shared plugin-data volume and static config mount when isLegacy is true', () => { + const compose = buildAppComposeContent(false, { isLegacy: true }); + expect(compose).toContain('plugin-data:/app/.plugins'); + expect(compose).toContain('plugin-data:'); + expect(compose).toContain('./simplens.config.yaml:/app/simplens.config.yaml:ro'); + expect(compose).toContain('SIMPLENS_CONFIG_PATH'); + }); + + it('generates new compose (> 1.3.0) without plugin-data volume and without config mount when plugins skipped', () => { + const compose = buildAppComposeContent(false, { isLegacy: false, hasPluginsConfig: false }); + expect(compose).not.toContain('plugin-data'); + expect(compose).not.toContain('simplens.config.yaml'); + expect(compose).not.toContain('SIMPLENS_CONFIG_PATH'); + expect(compose).toContain('logs-data:'); + }); + + it('generates new compose (> 1.3.0) with config mount and SIMPLENS_CONFIG_PATH when plugins are configured', () => { + const compose = buildAppComposeContent(false, { isLegacy: false, hasPluginsConfig: true }); + expect(compose).not.toContain('plugin-data'); + expect(compose).toContain('./simplens.config.yaml:/app/simplens.config.yaml:ro'); + expect(compose).toContain('SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml}'); + expect(compose).toContain('logs-data:'); + }); }); diff --git a/packages/onboard/src/config/constants.ts b/packages/onboard/src/config/constants.ts index fd808ca..59357fa 100644 --- a/packages/onboard/src/config/constants.ts +++ b/packages/onboard/src/config/constants.ts @@ -35,17 +35,28 @@ export const DOCKER_TIMEOUTS = { } as const; /** - * Critical environment variables that always need user input + * Legacy critical environment variables (SimpleNS <= 1.3.0) */ -export const CRITICAL_ENV_KEYS = [ +export const LEGACY_CRITICAL_ENV_KEYS = [ 'NS_API_KEY', 'MONGO_URI', 'BROKERS', 'REDIS_URL', 'AUTH_SECRET', 'ADMIN_PASSWORD', - 'CORE_VERSION', - 'DASHBOARD_VERSION', + 'VERSION', +]; + +/** + * Critical environment variables that always need user input (SimpleNS > 1.3.0) + */ +export const CRITICAL_ENV_KEYS = [ + 'NS_API_KEY', + 'MONGO_URI', + 'BROKERS', + 'REDIS_URL', + 'AUTH_SECRET', + 'VERSION', ]; /** diff --git a/packages/onboard/src/env-config.ts b/packages/onboard/src/env-config.ts index 7533e26..afb7eca 100644 --- a/packages/onboard/src/env-config.ts +++ b/packages/onboard/src/env-config.ts @@ -3,11 +3,36 @@ import { text, password } from '@clack/prompts'; import { handleCancel } from './ui.js'; import path from 'path'; import crypto from 'crypto'; -import { CRITICAL_ENV_KEYS } from './config/constants.js'; +import { CRITICAL_ENV_KEYS, LEGACY_CRITICAL_ENV_KEYS } from './config/constants.js'; import type { EnvVariable } from './types/domain.js'; export const DEFAULT_BASE_PATH = ''; -const AUTO_GENERATE_ON_EMPTY_KEYS = ['NS_API_KEY', 'AUTH_SECRET', 'CORE_VERSION', 'DASHBOARD_VERSION'] as const; + +/** + * Check if the provided SimpleNS version is greater than 1.3.0. + * Versions <= 1.3.0 are considered legacy. + * 'latest', 'master', empty/undefined, or non-numeric tags default to true (> 1.3.0). + */ +export function isVersionGreaterThan130(version: string | undefined): boolean { + if (!version || version === 'latest' || version === 'master') { + return true; + } + const clean = version.trim().replace(/^v/, ''); + const match = clean.match(/^(\d+)\.(\d+)(?:\.(\d+))?/); + if (!match) { + return true; + } + const major = parseInt(match[1], 10); + const minor = parseInt(match[2], 10); + const patch = match[3] !== undefined ? parseInt(match[3], 10) : 0; + + if (major > 1) return true; + if (major < 1) return false; + if (minor > 3) return true; + if (minor < 3) return false; + return patch > 0; +} +const AUTO_GENERATE_ON_EMPTY_KEYS = ['NS_API_KEY', 'AUTH_SECRET', 'VERSION'] as const; function shouldAutoGenerateOnEmpty(key: string, fullMode: boolean): boolean { return !fullMode && AUTO_GENERATE_ON_EMPTY_KEYS.includes(key as (typeof AUTO_GENERATE_ON_EMPTY_KEYS)[number]); @@ -41,7 +66,7 @@ export function generateDefaultValue(key: string): string { if (key === 'ADMIN_PASSWORD') { return `Admin${generateSecureRandom(16)}`; } - if (key === 'CORE_VERSION' || key ==='DASHBOARD_VERSION') { + if (key === 'VERSION') { return `latest`; } return ''; @@ -101,12 +126,55 @@ export async function promptBasePath(defaultValue: string = DEFAULT_BASE_PATH): return normalizeBasePath(result as string); } -/** - * Load and parse .env.example from embedded template - */ -export async function loadEnvExample(): Promise { - // Embedded .env template - always available regardless of installation - const envTemplate = ` +export const ESSENTIAL_ENV_TEMPLATE = ` +NODE_ENV=production +# ============================================ +# API SERVER +# ============================================ +NS_API_KEY= +PORT=3000 + +# ============================================ +# DATABASE +# ============================================ +MONGO_URI= + +# ============================================ +# KAFKA +# ============================================ +BROKERS= + +# ============================================ +# REDIS +# ============================================ +REDIS_URL= + +# ============================================ +# LOGGING (Grafana/Loki) +# ============================================ +LOKI_URL= +LOG_LEVEL=info +LOG_TO_FILE=true + +# ============================================ +# DOCKER IMAGE VERSION +# ============================================ +VERSION=latest + +# ============================================ +# ADMIN DASHBOARD +# ============================================ +AUTH_TRUST_HOST=true +AUTH_SECRET= +API_BASE_URL=http://api:3000 +WEBHOOK_HOST=dashboard +WEBHOOK_PORT=3002 +DASHBOARD_PORT=3002 +BASE_PATH= +HTTPS_COOKIE=false +`; + +export const LEGACY_ENV_TEMPLATE = ` NODE_ENV=production # ============================================ # API SERVER @@ -186,10 +254,9 @@ LOG_LEVEL=info LOG_TO_FILE=true # ============================================ -# DOCKER IMAGE VERSIONS +# DOCKER IMAGE VERSION # ============================================ -CORE_VERSION=latest -DASHBOARD_VERSION=latest +VERSION=latest # ============================================ # ADMIN DASHBOARD @@ -205,13 +272,10 @@ BASE_PATH= DASHBOARD_PORT=3002 `; - return parseEnvContent(envTemplate); -} - /** * Parse .env content into structured format */ -function parseEnvContent(content: string): EnvVariable[] { +function parseEnvContent(content: string, criticalKeys: readonly string[] = CRITICAL_ENV_KEYS): EnvVariable[] { const lines = content.split('\n'); const variables: EnvVariable[] = []; let currentComment = ''; @@ -237,9 +301,9 @@ function parseEnvContent(content: string): EnvVariable[] { const [, key, value] = match; variables.push({ key, - value: value || '', + value: value ? value.trim() : '', description: currentComment || undefined, - required: CRITICAL_ENV_KEYS.includes(key) || !value, + required: criticalKeys.includes(key) || !value, }); currentComment = ''; } @@ -248,6 +312,15 @@ function parseEnvContent(content: string): EnvVariable[] { return variables; } +/** + * Load and parse .env.example from embedded template + */ +export async function loadEnvExample(isLegacy: boolean = false): Promise { + const template = isLegacy ? LEGACY_ENV_TEMPLATE : ESSENTIAL_ENV_TEMPLATE; + const criticalKeys = isLegacy ? LEGACY_CRITICAL_ENV_KEYS : CRITICAL_ENV_KEYS; + return parseEnvContent(template, criticalKeys); +} + /** * Prompts user for environment variable values based on the selected mode. * @@ -262,11 +335,15 @@ export async function promptEnvVariables( infraServices: string[], basePath: string = DEFAULT_BASE_PATH, fullMode: boolean = false, - envOverrides: Partial> = {} + envOverrides: Partial> = {}, + version?: string ): Promise> { logInfo('Configuring environment variables...'); - const envVars = await loadEnvExample(); + const effectiveVersion = envOverrides.VERSION || envOverrides.CORE_VERSION || envOverrides.DASHBOARD_VERSION || version || 'latest'; + const isLegacy = !isVersionGreaterThan130(effectiveVersion); + const criticalKeys = isLegacy ? LEGACY_CRITICAL_ENV_KEYS : CRITICAL_ENV_KEYS; + const envVars = await loadEnvExample(isLegacy); const result = new Map(); const normalizedBasePath = normalizeBasePath(basePath); const basePathLabel = normalizedBasePath || '(root)'; @@ -291,8 +368,8 @@ export async function promptEnvVariables( if (mode === 'default') { // Use defaults, only prompt for critical values for (const envVar of envVars) { - if (fullMode && envVar.key in envOverrides) { - const overrideValue = envOverrides[envVar.key as keyof typeof envOverrides]; + if (fullMode && (envVar.key in envOverrides || (envVar.key === 'VERSION' && ('CORE_VERSION' in envOverrides || 'VERSION' in envOverrides)))) { + const overrideValue = envOverrides[envVar.key as keyof typeof envOverrides] || (envVar.key === 'VERSION' ? (envOverrides.VERSION || envOverrides.CORE_VERSION) : undefined); if (overrideValue) { result.set(envVar.key, overrideValue); continue; @@ -311,14 +388,20 @@ export async function promptEnvVariables( continue; } + // VERSION collected upfront or overridden + if (envVar.key === 'VERSION') { + result.set(envVar.key, effectiveVersion); + continue; + } + // Use default value if available - if (envVar.value && !CRITICAL_ENV_KEYS.includes(envVar.key)) { + if (envVar.value && !criticalKeys.includes(envVar.key)) { result.set(envVar.key, envVar.value); continue; } // Prompt for critical values (only if not auto-filled) - if (CRITICAL_ENV_KEYS.includes(envVar.key)) { + if (criticalKeys.includes(envVar.key)) { if (fullMode) { // In full mode, auto-generate critical values const defaultValue = generateDefaultValue(envVar.key); @@ -327,7 +410,7 @@ export async function promptEnvVariables( } } else { // Show changelog info for version variables - if (envVar.key === 'CORE_VERSION' || envVar.key === 'DASHBOARD_VERSION') { + if (envVar.key === 'VERSION') { logInfo('ℹ️ Visit https://simplens.in/changelog for version information'); } @@ -375,8 +458,8 @@ export async function promptEnvVariables( logInfo('Interactive mode: You will be prompted for each environment variable.'); for (const envVar of envVars) { - if (fullMode && envVar.key in envOverrides) { - const overrideValue = envOverrides[envVar.key as keyof typeof envOverrides]; + if (fullMode && (envVar.key in envOverrides || (envVar.key === 'VERSION' && ('CORE_VERSION' in envOverrides || 'VERSION' in envOverrides)))) { + const overrideValue = envOverrides[envVar.key as keyof typeof envOverrides] || (envVar.key === 'VERSION' ? (envOverrides.VERSION || envOverrides.CORE_VERSION) : undefined); if (overrideValue) { result.set(envVar.key, overrideValue); continue; @@ -390,9 +473,15 @@ export async function promptEnvVariables( result.set(envVar.key, normalizedBasePath); continue; } + + // VERSION collected upfront or overridden + if (envVar.key === 'VERSION') { + result.set(envVar.key, effectiveVersion); + continue; + } // Show changelog info for version variables - if (envVar.key === 'CORE_VERSION' || envVar.key === 'DASHBOARD_VERSION') { + if (envVar.key === 'VERSION') { logInfo('ℹ️ Visit https://simplens.in/changelog for version information'); } @@ -452,7 +541,7 @@ function getSuggestedValue(key: string): string { if (key === 'ADMIN_USERNAME') { return 'admin'; } - if (key === 'CORE_VERSION' || key === 'DASHBOARD_VERSION') { + if (key === 'VERSION' || key === 'CORE_VERSION' || key === 'DASHBOARD_VERSION') { return 'latest'; } return ''; @@ -463,12 +552,23 @@ function getSuggestedValue(key: string): string { */ export async function generateEnvFile( targetDir: string, - envVars: Map + envVars: Map, + isLegacy: boolean = false ): Promise { const envPath = path.join(targetDir, '.env'); let content = '# SimpleNS Environment Configuration\n'; - content += '# Generated by @simplens/onboard\n\n'; + content += '# Generated by @simplens/onboard\n'; + if (!isLegacy) { + content += '#\n'; + content += '# NOTE: All operational settings (worker batch sizes, poll intervals, retries,\n'; + content += '# deduplication TTLs, recovery thresholds, and log levels) are now managed\n'; + content += '# dynamically via MongoDB and the Admin Dashboard with live zero-downtime hot-reloading!\n'; + content += '#\n'; + content += '# Administrator credentials are configured interactively on first launch via\n'; + content += '# the Dashboard Setup UI (/setup) and stored securely in MongoDB.\n'; + } + content += '\n'; for (const [key, value] of envVars.entries()) { content += `${key}=${value}\n`; diff --git a/packages/onboard/src/index.ts b/packages/onboard/src/index.ts index 2a02efc..e028f54 100644 --- a/packages/onboard/src/index.ts +++ b/packages/onboard/src/index.ts @@ -36,6 +36,7 @@ import { normalizeBasePath, validateBasePath, DEFAULT_BASE_PATH, + isVersionGreaterThan130, } from './env-config.js'; import { fetchAvailablePlugins, @@ -62,15 +63,18 @@ const program = new Command(); program .name('@simplens/onboard') .description('A CLI tool to setup a SimpleNS instance on your machine/server') - .version('1.0.0') + .version('1.0.13') .option('--full', 'Non-interactive mode - all options must be provided via CLI') .option('--infra [services...]', 'Infrastructure services (mongo, kafka, kafka-ui, redis, nginx, loki, grafana)') .option('--env ', 'Environment setup mode: "default" or "interactive"') .option('--dir ', 'Target directory for setup') .option('--base-path ', 'Dashboard BASE_PATH (example: /dashboard, default: root)') - .option('--core-version ', 'Override CORE_VERSION in generated .env (primarily for --full mode)') - .option('--dashboard-version ', 'Override DASHBOARD_VERSION in generated .env (primarily for --full mode)') + .option('--app-version ', 'Override VERSION in generated .env (primarily for --full mode)') + .option('--version-tag ', 'Override VERSION in generated .env (alias)') + .option('--core-version ', 'Alias for --app-version / VERSION') + .option('--dashboard-version ', 'Alias for --app-version / VERSION') .option('--plugin [plugins...]', 'Plugins to install (e.g., @simplens/mock @simplens/nodemailer-gmail)') + .option('--skip-plugins', 'Skip plugin configuration (plugins can be installed via dashboard later)') .option('--ssl', 'Enable optional SSL certificate setup using Dockerized Certbot') .option('--ssl-domain ', 'Public domain for SSL certificate (required with --ssl in --full mode)') .option('--ssl-email ', 'Email for Let\'s Encrypt registration (required with --ssl in --full mode)') @@ -86,6 +90,8 @@ interface OnboardSetupOptions { enableSsl: boolean; sslDomain?: string; sslEmail?: string; + version: string; + skipPlugins?: boolean; } function printStep(step: number, total: number, title: string): void { @@ -134,6 +140,7 @@ function showSetupSummary(setupOptions: OnboardSetupOptions, targetDir: string, const summaryLines = [ `Target directory : ${targetDir}`, + `SimpleNS Version : ${setupOptions.version}`, `Infrastructure : ${infraLabel}`, `Environment mode : ${setupOptions.envMode}`, `BASE_PATH : ${basePathLabel}`, @@ -373,6 +380,27 @@ async function promptSetupOptions(options: any): Promise { } } + // --- Version --- + let versionValue: string; + const versionOption = (typeof options.appVersion === 'string' && options.appVersion.trim()) || + (typeof options.versionTag === 'string' && options.versionTag.trim()) || + (typeof options.coreVersion === 'string' && options.coreVersion.trim()) || + (typeof options.dashboardVersion === 'string' && options.dashboardVersion.trim()); + if (versionOption) { + versionValue = versionOption; + } else if (isFullMode) { + versionValue = 'latest'; + } else { + const result = await text({ + message: 'SimpleNS version to deploy (e.g. latest, 1.4.0, 1.3.0):', + defaultValue: 'latest', + placeholder: 'latest', + withGuide: true, + }); + handleCancel(result); + versionValue = (result as string).trim() || 'latest'; + } + return { infra: infraValue, infraServices: infraServices, @@ -383,6 +411,8 @@ async function promptSetupOptions(options: any): Promise { enableSsl: enableSslValue, sslDomain: sslDomainValue, sslEmail: sslEmailValue, + version: versionValue, + skipPlugins: options.skipPlugins === true, }; } @@ -416,6 +446,7 @@ async function main() { // Prompt for setup options if not provided const setupOptions = await promptSetupOptions(options); + const isLegacy = !isVersionGreaterThan130(setupOptions.version); // Get target directory const targetDir = path.resolve(setupOptions.targetDir); @@ -475,6 +506,8 @@ async function main() { await writeAppCompose(targetDir, { includeNginx: false, includeSsl: false, + isLegacy, + hasPluginsConfig: isLegacy ? true : (setupOptions.plugins.length > 0), }); const controlScriptFilename = await generateControlScript(targetDir, { os: detectOS(), @@ -484,10 +517,10 @@ async function main() { // Step 4: Environment configuration log.step('Step 4/6 — Environment Configuration'); const envMode = setupOptions.envMode; + const versionOverride = options.appVersion || options.versionTag || options.coreVersion || options.dashboardVersion; const envOverrides = options.full ? { - CORE_VERSION: options.coreVersion, - DASHBOARD_VERSION: options.dashboardVersion, + VERSION: versionOverride || setupOptions.version, } : undefined; const envVars = await promptEnvVariables( @@ -495,16 +528,24 @@ async function main() { selectedInfraServices, setupOptions.basePath, options.full || false, - envOverrides + envOverrides, + setupOptions.version ); - await generateEnvFile(targetDir, envVars); + await generateEnvFile(targetDir, envVars, isLegacy); // In full mode, notify user about auto-generated credentials if (options.full) { - logWarning( - '⚠️ Auto-generated credentials in .env file. ' + - 'Please update NS_API_KEY, AUTH_SECRET, and ADMIN_PASSWORD before deploying to production!' - ); + if (isLegacy) { + logWarning( + '⚠️ Auto-generated credentials in .env file. ' + + 'Please update NS_API_KEY, AUTH_SECRET, and ADMIN_PASSWORD before deploying to production!' + ); + } else { + logWarning( + '⚠️ Auto-generated credentials in .env file. ' + + 'Please update NS_API_KEY and AUTH_SECRET before deploying to production!' + ); + } } // Generate nginx.conf whenever nginx is active in either compose file @@ -522,14 +563,36 @@ async function main() { let selectedPlugins: string[] = []; let pluginCredentialKeys: string[] = []; - // Use pre-provided plugins from CLI, or prompt for them - if (setupOptions.plugins.length > 0) { - selectedPlugins = setupOptions.plugins; - log.info(`Using plugins: ${selectedPlugins.join(', ')}`); - } else if (!options.full) { - // Only prompt in interactive mode - const availablePlugins = await fetchAvailablePlugins(); - selectedPlugins = await promptPluginSelection(availablePlugins); + if (isLegacy) { + // Legacy SimpleNS <= 1.3.0: plugins always configured + if (setupOptions.plugins.length > 0) { + selectedPlugins = setupOptions.plugins; + log.info(`Using plugins: ${selectedPlugins.join(', ')}`); + } else if (!options.full) { + const availablePlugins = await fetchAvailablePlugins(); + selectedPlugins = await promptPluginSelection(availablePlugins); + } + } else { + // SimpleNS > 1.3.0: plugins are completely optional + if (setupOptions.plugins.length > 0) { + selectedPlugins = setupOptions.plugins; + log.info(`Using plugins: ${selectedPlugins.join(', ')}`); + } else if (setupOptions.skipPlugins || options.full) { + log.info('Skipping plugin configuration. You can install plugins via the Admin Dashboard.'); + } else { + const shouldConfigure = await confirm({ + message: 'Do you want to configure notification plugins now? (Skip to install later via Dashboard)', + initialValue: false, + withGuide: true, + }); + handleCancel(shouldConfigure); + if (shouldConfigure) { + const availablePlugins = await fetchAvailablePlugins(); + selectedPlugins = await promptPluginSelection(availablePlugins); + } else { + log.info('Skipping plugin configuration. You can install plugins via the Admin Dashboard.'); + } + } } if (selectedPlugins.length > 0) { @@ -554,6 +617,24 @@ async function main() { await appendPluginEnv(targetDir, pluginCreds); } } + + // For SimpleNS > 1.3.0, attach simplens.config.yaml mount and SIMPLENS_CONFIG_PATH + if (!isLegacy) { + await writeAppCompose(targetDir, { + includeNginx: false, + includeSsl: false, + isLegacy: false, + hasPluginsConfig: true, + }); + } + } else if (!isLegacy) { + // SimpleNS > 1.3.0 with no plugins selected: ensure compose file has no plugin mounts + await writeAppCompose(targetDir, { + includeNginx: false, + includeSsl: false, + isLegacy: false, + hasPluginsConfig: false, + }); } // Step 6: Service orchestration @@ -658,22 +739,26 @@ async function main() { const credentialWarnings = [ ' • NS_API_KEY - API authentication key', ' • AUTH_SECRET - Session secret for dashboard', - ' • ADMIN_PASSWORD - Dashboard admin password', ]; + if (isLegacy) { + credentialWarnings.push(' • ADMIN_PASSWORD - Dashboard admin password'); + } if (pluginCredentialKeys.length > 0) { credentialWarnings.push(` • Plugin credentials: ${pluginCredentialKeys.join(', ')}`); } - note( - '⚠️ IMPORTANT: Auto-generated credentials were used for non-interactive setup.\n' + - '\n' + + let securityNotice = '⚠️ IMPORTANT: Auto-generated credentials were used for non-interactive setup.\n\n' + 'Please update the following in your .env file before production use:\n' + credentialWarnings.join('\n') + '\n\n' + - 'Default credentials are NOT secure for production environments.', - 'Security Notice' - ); + 'Default credentials are NOT secure for production environments.'; + + if (!isLegacy) { + securityNotice += '\n\nAdministrator credentials are set up upon first launch in the Dashboard at /setup.'; + } + + note(securityNotice, 'Security Notice'); } // Display access information @@ -696,6 +781,13 @@ async function main() { ); } + if (!isLegacy) { + note( + 'First-time setup: Open the Dashboard in your browser to configure your administrator account (/setup).', + 'Next Steps' + ); + } + // Clack outro outro('Setup complete — happy notifying! 🚀'); diff --git a/packages/onboard/src/infra.ts b/packages/onboard/src/infra.ts index fc83c8d..2594d67 100644 --- a/packages/onboard/src/infra.ts +++ b/packages/onboard/src/infra.ts @@ -1,5 +1,6 @@ import { APP_COMPOSE_TEMPLATE, + LEGACY_APP_COMPOSE_TEMPLATE, APP_NGINX_SERVICE_TEMPLATE, APP_NGINX_SSL_SERVICE_TEMPLATE, APP_CERTBOT_SERVICES_TEMPLATE, @@ -305,15 +306,22 @@ export async function generateInfraCompose( s.stop('Generated docker-compose.infra.yaml'); } +export interface AppComposeOptions { + includeSsl?: boolean; + isLegacy?: boolean; + hasPluginsConfig?: boolean; +} + /** * Build app docker-compose content. * Optionally inject nginx reverse-proxy service before the volumes section. */ export function buildAppComposeContent( includeNginx: boolean, - options: { includeSsl?: boolean } = {} + options: AppComposeOptions = {} ): string { - let content = APP_COMPOSE_TEMPLATE; + const isLegacy = options.isLegacy === true; + let content = isLegacy ? LEGACY_APP_COMPOSE_TEMPLATE : APP_COMPOSE_TEMPLATE; const includeSsl = options.includeSsl === true; const shouldIncludeNginx = includeNginx || includeSsl; const marker = '\nvolumes:'; @@ -322,6 +330,16 @@ export function buildAppComposeContent( return content; } + if (!isLegacy && options.hasPluginsConfig === true) { + const apiMarker = ' api:\n image: ghcr.io/simplenotificationsystem/simplens-core:${VERSION:-latest}\n container_name: api\n ports:\n - 3000:3000\n env_file:\n - .env\n volumes:\n - logs-data:/app/logs'; + const apiReplacement = ' api:\n image: ghcr.io/simplenotificationsystem/simplens-core:${VERSION:-latest}\n container_name: api\n ports:\n - 3000:3000\n env_file:\n - .env\n environment:\n SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml}\n volumes:\n - logs-data:/app/logs\n - ./simplens.config.yaml:/app/simplens.config.yaml:ro'; + content = content.replace(apiMarker, apiReplacement); + + const npMarker = ' notification_processor:\n image: ghcr.io/simplenotificationsystem/simplens-core:${VERSION:-latest}\n container_name: notification_processor\n env_file:\n - .env\n volumes:\n - logs-data:/app/logs'; + const npReplacement = ' notification_processor:\n image: ghcr.io/simplenotificationsystem/simplens-core:${VERSION:-latest}\n container_name: notification_processor\n env_file:\n - .env\n environment:\n SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml}\n volumes:\n - logs-data:/app/logs\n - ./simplens.config.yaml:/app/simplens.config.yaml:ro'; + content = content.replace(npMarker, npReplacement); + } + if (shouldIncludeNginx) { const nginxBlock = includeSsl ? APP_NGINX_SSL_SERVICE_TEMPLATE @@ -331,10 +349,17 @@ export function buildAppComposeContent( if (includeSsl) { content = content.replace(marker, `\n${APP_CERTBOT_SERVICES_TEMPLATE}\n${marker}`); - content = content.replace( - '\nvolumes:\n plugin-data:', - '\nvolumes:\n certbot-etc:\n certbot-www:\n plugin-data:' - ); + if (isLegacy) { + content = content.replace( + '\nvolumes:\n plugin-data:', + '\nvolumes:\n certbot-etc:\n certbot-www:\n plugin-data:' + ); + } else { + content = content.replace( + '\nvolumes:\n logs-data:', + '\nvolumes:\n certbot-etc:\n certbot-www:\n logs-data:' + ); + } } return content; @@ -345,13 +370,20 @@ export function buildAppComposeContent( */ export async function writeAppCompose( targetDir: string, - options: { includeNginx?: boolean; includeSsl?: boolean } = {} + options: { + includeNginx?: boolean; + includeSsl?: boolean; + isLegacy?: boolean; + hasPluginsConfig?: boolean; + } = {} ): Promise { const s = spinner(); s.start('Generating docker-compose.yaml...'); const appPath = path.join(targetDir, 'docker-compose.yaml'); const appContent = buildAppComposeContent(options.includeNginx === true, { includeSsl: options.includeSsl === true, + isLegacy: options.isLegacy === true, + hasPluginsConfig: options.hasPluginsConfig === true, }); await writeFile(appPath, appContent); s.stop('Generated docker-compose.yaml'); diff --git a/packages/onboard/src/templates.ts b/packages/onboard/src/templates.ts index cd7b9d3..5b06a19 100644 --- a/packages/onboard/src/templates.ts +++ b/packages/onboard/src/templates.ts @@ -119,9 +119,9 @@ volumes: grafana_data: `; -export const APP_COMPOSE_TEMPLATE = `services: +export const LEGACY_APP_COMPOSE_TEMPLATE = `services: api: - image: ghcr.io/simplenotificationsystem/simplens-core:\${CORE_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} container_name: api ports: - 3000:3000 @@ -143,7 +143,7 @@ export const APP_COMPOSE_TEMPLATE = `services: start_period: 10s worker: - image: ghcr.io/simplenotificationsystem/simplens-core:\${CORE_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} env_file: - .env volumes: @@ -152,7 +152,7 @@ export const APP_COMPOSE_TEMPLATE = `services: restart: unless-stopped notification_processor: - image: ghcr.io/simplenotificationsystem/simplens-core:\${CORE_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} env_file: - .env environment: @@ -168,7 +168,7 @@ export const APP_COMPOSE_TEMPLATE = `services: restart: unless-stopped delayed_processor: - image: ghcr.io/simplenotificationsystem/simplens-core:\${CORE_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} env_file: - .env volumes: @@ -177,7 +177,7 @@ export const APP_COMPOSE_TEMPLATE = `services: restart: unless-stopped recovery: - image: ghcr.io/simplenotificationsystem/simplens-core:\${CORE_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} env_file: - .env volumes: @@ -186,7 +186,7 @@ export const APP_COMPOSE_TEMPLATE = `services: restart: unless-stopped dashboard: - image: ghcr.io/simplenotificationsystem/simplens-dashboard:\${DASHBOARD_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-dashboard:\${VERSION:-latest} ports: - 3002:3002 container_name: dashboard @@ -208,6 +208,102 @@ networks: name: simplens `; +export const APP_COMPOSE_TEMPLATE = `services: + api: + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} + container_name: api + ports: + - 3000:3000 + env_file: + - .env + volumes: + - logs-data:/app/logs + command: [ "node", "dist/api/server.js" ] + restart: unless-stopped + healthcheck: + test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + worker: + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} + container_name: worker + env_file: + - .env + volumes: + - logs-data:/app/logs + command: [ "node", "dist/workers/worker.js" ] + depends_on: + api: + condition: service_healthy + restart: unless-stopped + + notification_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} + container_name: notification_processor + env_file: + - .env + volumes: + - logs-data:/app/logs + command: [ "node", "dist/processors/unified/unified.processor.js" ] + depends_on: + api: + condition: service_healthy + restart: unless-stopped + + delayed_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} + container_name: delayed_processor + env_file: + - .env + volumes: + - logs-data:/app/logs + command: [ "node", "dist/processors/delayed/delayed.processor.js" ] + depends_on: + api: + condition: service_healthy + restart: unless-stopped + + recovery: + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} + container_name: recovery + env_file: + - .env + volumes: + - logs-data:/app/logs + command: [ "node", "dist/workers/recovery/recovery.service.js" ] + depends_on: + api: + condition: service_healthy + restart: unless-stopped + + dashboard: + image: ghcr.io/simplenotificationsystem/simplens-dashboard:\${VERSION:-latest} + ports: + - 3002:3002 + container_name: dashboard + env_file: + - .env + environment: + PORT: \${DASHBOARD_PORT:-3002} + API_BASE_URL: http://api:\${PORT:-3000} + WEBHOOK_HOST: dashboard + WEBHOOK_PORT: \${DASHBOARD_PORT:-3002} + depends_on: + api: + condition: service_healthy + restart: unless-stopped + +volumes: + logs-data: + +networks: + default: + name: simplens +`; + export const APP_NGINX_SERVICE_TEMPLATE = ` nginx: image: nginx:alpine container_name: nginx From 13a86475ebcfd2150a18aa3cc91088ba02fefc04 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Wed, 23 Sep 2026 06:32:33 +0530 Subject: [PATCH 28/46] feat: expand tool suite to 54 tools, add channel routing and provider management features --- packages/mcp-server/README.md | 50 +++- packages/mcp-server/package-lock.json | 4 +- packages/mcp-server/package.json | 2 +- packages/mcp-server/src/api-client.ts | 250 ++++++++++++++++++ packages/mcp-server/src/tools/admin-auth.ts | 29 ++ .../mcp-server/src/tools/channel-routing.ts | 97 +++++++ packages/mcp-server/src/tools/index.ts | 10 + .../src/tools/plugins-management.ts | 183 +++++++++++++ packages/mcp-server/src/tools/providers.ts | 208 +++++++++++++++ packages/mcp-server/src/tools/settings.ts | 70 +++++ packages/mcp-server/tests/verify-stdio.ts | 4 +- 11 files changed, 900 insertions(+), 7 deletions(-) create mode 100644 packages/mcp-server/src/tools/admin-auth.ts create mode 100644 packages/mcp-server/src/tools/channel-routing.ts create mode 100644 packages/mcp-server/src/tools/plugins-management.ts create mode 100644 packages/mcp-server/src/tools/providers.ts create mode 100644 packages/mcp-server/src/tools/settings.ts diff --git a/packages/mcp-server/README.md b/packages/mcp-server/README.md index 644f1b3..3241b12 100644 --- a/packages/mcp-server/README.md +++ b/packages/mcp-server/README.md @@ -8,7 +8,7 @@ A Model Context Protocol (MCP) server for the SimpleNS notification orchestratio - **Streamable HTTP Transport**: Connect remote or local MCP clients via HTTP (`/mcp`). - **Stdio Transport**: Local execution via command line (`--stdio`). -- **Full Tool Suite (29 Tools)**: Comprehensive coverage for notifications, templates, alerts, logs, metrics, and channel configuration. +- **Full Tool Suite (54 Tools)**: Comprehensive coverage for notifications, templates, alerts, logs, metrics, channel routing, providers, rate limits, dynamic settings, plugin lifecycle, and admin configuration. --- @@ -87,7 +87,7 @@ Add to your MCP Client configuration: --- -## Complete Tools Reference (29 Tools) +## Complete Tools Reference (54 Tools) ### 1. Notification Dispatch | Tool | Description | @@ -145,3 +145,49 @@ Add to your MCP Client configuration: | `delete_admin_channel` | Delete an admin channel configuration. | | `test_admin_channel` | Execute a live connection and credential test for an admin channel. | | `validate_admin_channel_config` | Validate configuration parameters against provider requirements before saving. | + +### 8. Channel Routing & Kafka Partition Scaling +| Tool | Description | +| :--- | :--- | +| `list_channel_routings` | List all configured channel routings showing default provider, fallback cascade, and partition counts. | +| `get_channel_routing` | Retrieve routing configuration and fallback chain for a specific channel (e.g. email, sms). | +| `set_channel_routing` | Configure default provider, ordered cascading fallback providers, and Kafka partitions for a channel. | +| `delete_channel_routing` | Remove routing rules for a channel. | + +### 9. Provider Management & Real-Time Rate Limiting +| Tool | Description | +| :--- | :--- | +| `list_providers` | List all configured provider instances with their enabled status and options. | +| `get_provider` | Retrieve detailed configuration and status for a single provider instance (supports optional decrypted view). | +| `create_provider` | Instantiate and register a new provider instance with credentials and rate limit options. | +| `update_provider` | Update credentials, options, or toggle enabled state of an existing provider. | +| `delete_provider` | Remove a provider instance (verifies no active channel routings depend on it). | +| `test_provider` | Test connection and credentials for an existing provider or unpersisted configuration. | +| `get_providers_rate_limits` | Real-time token bucket capacity, refill rates, and metrics across all providers. | +| `get_provider_rate_limit` | Real-time rate limit status and remaining token capacity for a specific provider instance. | +| `reset_provider_rate_limit` | Manually reset rate limiter tokens and telemetry counters for a specific provider in Redis. | + +### 10. Dynamic Operational Settings +| Tool | Description | +| :--- | :--- | +| `get_operational_settings` | Retrieve active operational settings (worker concurrency, retry limits, polling intervals, DLQ thresholds). | +| `update_operational_settings` | Hot-update operational settings cluster-wide via Redis pub/sub with zero service restarts. | +| `reset_operational_settings` | Reset all dynamic operational configuration parameters to system defaults. | + +### 11. Plugin Lifecycle & NPM Registry Management +| Tool | Description | +| :--- | :--- | +| `list_installed_plugins` | List all installed npm plugin packages stored in MongoDB with version and installation timestamp. | +| `get_plugin_catalog` | Browse available plugins from the public catalog by category (`official` or `community`). | +| `install_plugin` | Download, validate, and hot-load an npm provider plugin package with optional private auth. | +| `change_plugin_version` | Upgrade or downgrade an installed plugin to a specified semver version. | +| `uninstall_plugin` | Uninstall an installed plugin package and clean up associated resources. | +| `get_npm_auth` | Check configured npm registry authentication status and masked tokens. | +| `save_npm_auth` | Save private npm registry credentials (auth token, registry URL, optional scope). | +| `delete_npm_auth` | Remove saved npm registry authentication credentials. | + +### 12. Admin Setup Status +| Tool | Description | +| :--- | :--- | +| `get_admin_auth_status` | Check whether SimpleNS administrator credentials have been configured or require initial setup. | + diff --git a/packages/mcp-server/package-lock.json b/packages/mcp-server/package-lock.json index 68eb195..691bb63 100644 --- a/packages/mcp-server/package-lock.json +++ b/packages/mcp-server/package-lock.json @@ -1,12 +1,12 @@ { "name": "@simplens/mcp", - "version": "1.1.2", + "version": "1.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@simplens/mcp", - "version": "1.1.2", + "version": "1.2.0", "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "^1.12.1", diff --git a/packages/mcp-server/package.json b/packages/mcp-server/package.json index dce3fae..486d496 100644 --- a/packages/mcp-server/package.json +++ b/packages/mcp-server/package.json @@ -1,6 +1,6 @@ { "name": "@simplens/mcp", - "version": "1.1.2", + "version": "1.2.0", "description": "Remote MCP server for SimpleNS notification orchestration engine", "type": "module", "main": "dist/index.js", diff --git a/packages/mcp-server/src/api-client.ts b/packages/mcp-server/src/api-client.ts index 9b76265..fde3ff0 100644 --- a/packages/mcp-server/src/api-client.ts +++ b/packages/mcp-server/src/api-client.ts @@ -351,6 +351,256 @@ export class ApiClient { headers: this.authHeader, }); } + + // ============================================================================ + // CHANNEL ROUTING + // ============================================================================ + + /** GET /api/channels/routing - List all channel routings */ + async listChannelRoutings(): Promise { + return request(this.baseUrl, 'api/channels/routing', { + headers: this.authHeader, + }); + } + + /** GET /api/channels/routing/:channel - Get routing for a specific channel */ + async getChannelRouting(channel: string): Promise { + return request(this.baseUrl, `api/channels/routing/${encodeURIComponent(channel)}`, { + headers: this.authHeader, + }); + } + + /** PUT /api/channels/routing/:channel - Set or update routing for a channel */ + async setChannelRouting(channel: string, payload: { + default_provider_id: string; + fallback_provider_ids?: string[]; + partitions?: number; + }): Promise { + return request(this.baseUrl, `api/channels/routing/${encodeURIComponent(channel)}`, { + method: 'PUT', + headers: this.authHeader, + body: payload, + }); + } + + /** DELETE /api/channels/routing/:channel - Delete routing for a channel */ + async deleteChannelRouting(channel: string): Promise { + return request(this.baseUrl, `api/channels/routing/${encodeURIComponent(channel)}`, { + method: 'DELETE', + headers: this.authHeader, + }); + } + + // ============================================================================ + // PROVIDERS & RATE LIMITS + // ============================================================================ + + /** GET /api/providers - List configured providers */ + async listProviders(): Promise { + return request(this.baseUrl, 'api/providers', { + headers: this.authHeader, + }); + } + + /** GET /api/providers/:id - Get single provider by ID */ + async getProvider(id: string, includeDecrypted = false): Promise { + return request(this.baseUrl, `api/providers/${encodeURIComponent(id)}`, { + headers: this.authHeader, + params: includeDecrypted ? { include_decrypted: 'true' } : undefined, + }); + } + + /** POST /api/providers - Create a new provider */ + async createProvider(payload: { + id: string; + plugin_name: string; + credentials: Record; + options?: Record; + enabled?: boolean; + }): Promise { + return request(this.baseUrl, 'api/providers', { + method: 'POST', + headers: this.authHeader, + body: payload, + }); + } + + /** PUT /api/providers/:id - Update an existing provider */ + async updateProvider(id: string, payload: { + credentials?: Record; + options?: Record; + enabled?: boolean; + }): Promise { + return request(this.baseUrl, `api/providers/${encodeURIComponent(id)}`, { + method: 'PUT', + headers: this.authHeader, + body: payload, + }); + } + + /** DELETE /api/providers/:id - Delete a provider instance */ + async deleteProvider(id: string): Promise { + return request(this.baseUrl, `api/providers/${encodeURIComponent(id)}`, { + method: 'DELETE', + headers: this.authHeader, + }); + } + + /** POST /api/providers/test - Test provider connection/credentials */ + async testProvider(payload: { + provider_id?: string; + plugin_name?: string; + credentials?: Record; + options?: Record; + }): Promise { + return request(this.baseUrl, 'api/providers/test', { + method: 'POST', + headers: this.authHeader, + body: payload, + }); + } + + /** GET /api/providers/rate-limits - Get rate limits for all providers */ + async getProvidersRateLimits(): Promise { + return request(this.baseUrl, 'api/providers/rate-limits', { + headers: this.authHeader, + }); + } + + /** GET /api/providers/:id/rate-limit - Get rate limit status for a provider */ + async getProviderRateLimit(id: string): Promise { + return request(this.baseUrl, `api/providers/${encodeURIComponent(id)}/rate-limit`, { + headers: this.authHeader, + }); + } + + /** POST /api/providers/:id/rate-limit/reset - Reset rate limit for a provider */ + async resetProviderRateLimit(id: string): Promise { + return request(this.baseUrl, `api/providers/${encodeURIComponent(id)}/rate-limit/reset`, { + method: 'POST', + headers: this.authHeader, + }); + } + + // ============================================================================ + // OPERATIONAL DYNAMIC SETTINGS + // ============================================================================ + + /** GET /api/settings - Get operational settings */ + async getOperationalSettings(): Promise { + return request(this.baseUrl, 'api/settings', { + headers: this.authHeader, + }); + } + + /** PUT /api/settings - Update operational settings */ + async updateOperationalSettings(payload: Record): Promise { + return request(this.baseUrl, 'api/settings', { + method: 'PUT', + headers: this.authHeader, + body: payload, + }); + } + + /** POST /api/settings/reset - Reset operational settings to defaults */ + async resetOperationalSettings(): Promise { + return request(this.baseUrl, 'api/settings/reset', { + method: 'POST', + headers: this.authHeader, + }); + } + + // ============================================================================ + // EXTENDED PLUGINS & NPM AUTH + // ============================================================================ + + /** GET /api/plugins/installed - List installed plugins in MongoDB */ + async listInstalledPlugins(): Promise { + return request(this.baseUrl, 'api/plugins/installed', { + headers: this.authHeader, + }); + } + + /** GET /api/plugins/catalog/:category - Get plugin catalog */ + async getPluginCatalog(category: 'official' | 'community'): Promise { + return request(this.baseUrl, `api/plugins/catalog/${category}`, { + headers: this.authHeader, + }); + } + + /** POST /api/plugins/install - Install npm plugin package */ + async installPlugin(payload: { + package: string; + version?: string; + auth?: { token?: string; registry_url?: string }; + }): Promise { + return request(this.baseUrl, 'api/plugins/install', { + method: 'POST', + headers: this.authHeader, + body: payload, + }); + } + + /** PUT /api/plugins/version - Change version of installed plugin */ + async changePluginVersion(payload: { + package: string; + version: string; + }): Promise { + return request(this.baseUrl, 'api/plugins/version', { + method: 'PUT', + headers: this.authHeader, + body: payload, + }); + } + + /** DELETE /api/plugins - Uninstall plugin */ + async uninstallPlugin(packageName: string): Promise { + return request(this.baseUrl, 'api/plugins', { + method: 'DELETE', + headers: this.authHeader, + params: { package: packageName }, + }); + } + + /** GET /api/plugins/npm-auth - Get npm auth status */ + async getNpmAuth(): Promise { + return request(this.baseUrl, 'api/plugins/npm-auth', { + headers: this.authHeader, + }); + } + + /** POST /api/plugins/npm-auth - Save npm auth credentials */ + async saveNpmAuth(payload: { + token: string; + registry_url?: string; + scope?: string; + }): Promise { + return request(this.baseUrl, 'api/plugins/npm-auth', { + method: 'POST', + headers: this.authHeader, + body: payload, + }); + } + + /** DELETE /api/plugins/npm-auth - Delete npm auth configuration */ + async deleteNpmAuth(target?: string): Promise { + return request(this.baseUrl, 'api/plugins/npm-auth', { + method: 'DELETE', + headers: this.authHeader, + params: target ? { id: target } : undefined, + }); + } + + // ============================================================================ + // ADMIN AUTH STATUS + // ============================================================================ + + /** GET /api/admin/auth/status - Check if admin credentials are configured */ + async getAdminAuthStatus(): Promise { + return request(this.baseUrl, 'api/admin/auth/status', { + headers: this.authHeader, + }); + } } diff --git a/packages/mcp-server/src/tools/admin-auth.ts b/packages/mcp-server/src/tools/admin-auth.ts new file mode 100644 index 0000000..c70e30c --- /dev/null +++ b/packages/mcp-server/src/tools/admin-auth.ts @@ -0,0 +1,29 @@ +/** + * Tools: Admin Authentication & Setup Status + */ + +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { UserCredentials } from '../auth.js'; +import { ApiClient } from '../api-client.js'; +import { formatApiResponse, formatToolError } from './response.js'; + +export function registerAdminAuthTools(server: McpServer, getCredentials: () => UserCredentials) { + // 1. get_admin_auth_status + server.registerTool( + 'get_admin_auth_status', + { + description: 'Check whether SimpleNS administrator credentials have been configured, or whether the system is pending initial dashboard onboarding.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getAdminAuthStatus(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to check admin auth status', error); + } + } + ); +} diff --git a/packages/mcp-server/src/tools/channel-routing.ts b/packages/mcp-server/src/tools/channel-routing.ts new file mode 100644 index 0000000..14992d3 --- /dev/null +++ b/packages/mcp-server/src/tools/channel-routing.ts @@ -0,0 +1,97 @@ +/** + * Tools: Channel Routing Configuration + */ + +import { z } from 'zod'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { UserCredentials } from '../auth.js'; +import { ApiClient } from '../api-client.js'; +import { formatApiResponse, formatToolError } from './response.js'; + +export function registerChannelRoutingTools(server: McpServer, getCredentials: () => UserCredentials) { + // 1. list_channel_routings + server.registerTool( + 'list_channel_routings', + { + description: 'List all configured channel routings, showing the primary default provider, ordered cascading fallback providers, and Kafka partitions assigned to each channel (e.g. email, sms, whatsapp).', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.listChannelRoutings(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to list channel routings', error); + } + } + ); + + // 2. get_channel_routing + server.registerTool( + 'get_channel_routing', + { + description: 'Get routing configuration for a specific communication channel (e.g. "email", "sms", "whatsapp"). Returns default provider ID, fallback provider chain, and partition count.', + inputSchema: { + channel: z.string().describe('Channel identifier (e.g. "email", "sms", "whatsapp", "inapp")'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getChannelRouting(params.channel); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to get routing for channel '${params.channel}'`, error); + } + } + ); + + // 3. set_channel_routing + server.registerTool( + 'set_channel_routing', + { + description: 'Set or update provider routing for a channel. Specifies the default provider to attempt first, optional ordered fallback providers if the primary fails or rate limits, and optional Kafka partitions for scaling throughput.', + inputSchema: { + channel: z.string().describe('Channel identifier (e.g. "email", "sms", "whatsapp")'), + default_provider_id: z.string().describe('The primary provider ID configured in SimpleNS to route messages through'), + fallback_provider_ids: z.array(z.string()).optional().describe('Ordered list of fallback provider IDs to cascade to if the default provider fails or is rate limited'), + partitions: z.number().int().min(1).optional().describe('Number of Kafka topic partitions for this channel (can only be scaled up, not down)'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const { channel, ...payload } = params; + const result = await client.setChannelRouting(channel, payload); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to set routing for channel '${params.channel}'`, error); + } + } + ); + + // 4. delete_channel_routing + server.registerTool( + 'delete_channel_routing', + { + description: 'Delete routing configuration for a channel. Messages sent to this channel will fail until routing is re-configured.', + inputSchema: { + channel: z.string().describe('Channel identifier to delete routing for (e.g. "email", "sms")'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.deleteChannelRouting(params.channel); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to delete routing for channel '${params.channel}'`, error); + } + } + ); +} diff --git a/packages/mcp-server/src/tools/index.ts b/packages/mcp-server/src/tools/index.ts index 12e6367..1365c1f 100644 --- a/packages/mcp-server/src/tools/index.ts +++ b/packages/mcp-server/src/tools/index.ts @@ -20,6 +20,11 @@ import { registerNotificationsManagementTools } from './notifications-management import { registerResolveAlertsRetryTools } from './resolve-alerts-retry.js'; import { registerDashboardTools } from './dashboard.js'; import { registerAdminChannelsTools } from './admin-channels.js'; +import { registerChannelRoutingTools } from './channel-routing.js'; +import { registerProviderTools } from './providers.js'; +import { registerSettingsTools } from './settings.js'; +import { registerPluginsManagementTools } from './plugins-management.js'; +import { registerAdminAuthTools } from './admin-auth.js'; export function registerAllTools(server: McpServer, getCredentials: () => UserCredentials) { registerSendNotification(server, getCredentials); @@ -35,4 +40,9 @@ export function registerAllTools(server: McpServer, getCredentials: () => UserCr registerResolveAlertsRetryTools(server, getCredentials); registerDashboardTools(server, getCredentials); registerAdminChannelsTools(server, getCredentials); + registerChannelRoutingTools(server, getCredentials); + registerProviderTools(server, getCredentials); + registerSettingsTools(server, getCredentials); + registerPluginsManagementTools(server, getCredentials); + registerAdminAuthTools(server, getCredentials); } diff --git a/packages/mcp-server/src/tools/plugins-management.ts b/packages/mcp-server/src/tools/plugins-management.ts new file mode 100644 index 0000000..e427b61 --- /dev/null +++ b/packages/mcp-server/src/tools/plugins-management.ts @@ -0,0 +1,183 @@ +/** + * Tools: Extended Plugin Lifecycle & NPM Registry Management + */ + +import { z } from 'zod'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { UserCredentials } from '../auth.js'; +import { ApiClient } from '../api-client.js'; +import { formatApiResponse, formatToolError } from './response.js'; + +export function registerPluginsManagementTools(server: McpServer, getCredentials: () => UserCredentials) { + // 1. list_installed_plugins + server.registerTool( + 'list_installed_plugins', + { + description: 'List all installed npm plugin packages persisted in MongoDB with their installation timestamp, version, and status.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.listInstalledPlugins(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to list installed plugins', error); + } + } + ); + + // 2. get_plugin_catalog + server.registerTool( + 'get_plugin_catalog', + { + description: 'Browse the public SimpleNS plugin catalog to discover available provider plugins by category ("official" or "community").', + inputSchema: { + category: z.enum(['official', 'community']).describe('Plugin category: "official" for verified first-party plugins or "community" for community plugins'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getPluginCatalog(params.category); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to fetch ${params.category} plugin catalog`, error); + } + } + ); + + // 3. install_plugin + server.registerTool( + 'install_plugin', + { + description: 'Install an npm plugin package into SimpleNS. The core engine downloads, validates, and hot-loads the plugin across the cluster.', + inputSchema: { + package: z.string().describe('The npm package name (e.g. "@simplens/smtp", "@simplens/resend")'), + version: z.string().optional().describe('Optional specific semver version or tag (default: "latest")'), + auth: z.object({ + token: z.string().optional().describe('Authentication token for private npm registry or package'), + registry_url: z.string().optional().describe('Custom registry URL if not installing from the default npm registry'), + }).optional().describe('Optional authentication details for private registries or scoped packages'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.installPlugin(params); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to install plugin '${params.package}'`, error); + } + } + ); + + // 4. change_plugin_version + server.registerTool( + 'change_plugin_version', + { + description: 'Upgrade or downgrade an installed plugin to a specific npm version.', + inputSchema: { + package: z.string().describe('The installed npm package name'), + version: z.string().describe('The target semver version to upgrade or downgrade to (e.g. "1.2.0")'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.changePluginVersion(params); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to change version of plugin '${params.package}'`, error); + } + } + ); + + // 5. uninstall_plugin + server.registerTool( + 'uninstall_plugin', + { + description: 'Uninstall a plugin package from SimpleNS. Fails if existing providers depend on this plugin.', + inputSchema: { + package: z.string().describe('The npm package name to uninstall (e.g. "@simplens/resend")'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.uninstallPlugin(params.package); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to uninstall plugin '${params.package}'`, error); + } + } + ); + + // 6. get_npm_auth + server.registerTool( + 'get_npm_auth', + { + description: 'Retrieve configured npm registry authentication status and masked tokens.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getNpmAuth(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to retrieve npm auth configuration', error); + } + } + ); + + // 7. save_npm_auth + server.registerTool( + 'save_npm_auth', + { + description: 'Configure and save npm authentication credentials (auth token and optional registry URL or scope) for private package access.', + inputSchema: { + token: z.string().describe('NPM auth token'), + registry_url: z.string().optional().describe('Custom NPM registry URL (e.g. "https://registry.npmjs.org/" or private Verdaccio/Artifactory URL)'), + scope: z.string().optional().describe('Package scope to associate this auth token with (e.g. "@myorg")'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.saveNpmAuth(params); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to save npm auth configuration', error); + } + } + ); + + // 8. delete_npm_auth + server.registerTool( + 'delete_npm_auth', + { + description: 'Delete saved npm authentication credentials for a specific scope or registry.', + inputSchema: { + target: z.string().optional().describe('Scope or ID identifier to remove, or omit to clear configuration'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.deleteNpmAuth(params.target); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to delete npm auth configuration', error); + } + } + ); +} diff --git a/packages/mcp-server/src/tools/providers.ts b/packages/mcp-server/src/tools/providers.ts new file mode 100644 index 0000000..b03ab99 --- /dev/null +++ b/packages/mcp-server/src/tools/providers.ts @@ -0,0 +1,208 @@ +/** + * Tools: Provider Management & Rate Limiting + */ + +import { z } from 'zod'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { UserCredentials } from '../auth.js'; +import { ApiClient } from '../api-client.js'; +import { formatApiResponse, formatToolError } from './response.js'; + +export function registerProviderTools(server: McpServer, getCredentials: () => UserCredentials) { + // 1. list_providers + server.registerTool( + 'list_providers', + { + description: 'List all configured provider instances across all notification channels, including their plugin package name, enabled status, and provider options.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.listProviders(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to list providers', error); + } + } + ); + + // 2. get_provider + server.registerTool( + 'get_provider', + { + description: 'Retrieve detailed configuration and status for a single provider instance by its ID. Credentials are masked by default.', + inputSchema: { + id: z.string().describe('Unique identifier of the provider instance (e.g. "sendgrid-primary", "twilio-marketing")'), + include_decrypted: z.boolean().optional().describe('Whether to reveal decrypted credentials instead of masked values (default: false)'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getProvider(params.id, params.include_decrypted); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to get provider '${params.id}'`, error); + } + } + ); + + // 3. create_provider + server.registerTool( + 'create_provider', + { + description: 'Instantiate and register a new provider with required credentials, options (e.g. rate limits), and enable state.', + inputSchema: { + id: z.string().describe('Unique user-defined identifier for this provider instance (e.g. "sendgrid-backup", "resend-prod")'), + plugin_name: z.string().describe('The installed plugin package name that powers this provider (e.g. "@simplens/smtp", "@simplens/sendgrid")'), + credentials: z.record(z.string(), z.unknown()).describe('Credential object matching the plugin schema (e.g. { apiKey: "..." } or { host: "...", port: 587 })'), + options: z.record(z.string(), z.unknown()).optional().describe('Optional provider parameters such as rate limits { rateLimit: { windowMs: 60000, maxRequests: 100 } }'), + enabled: z.boolean().optional().describe('Whether the provider is immediately active and available for routing (default: true)'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.createProvider(params); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to create provider '${params.id}'`, error); + } + } + ); + + // 4. update_provider + server.registerTool( + 'update_provider', + { + description: 'Update credentials, options, or enabled status of an existing provider instance.', + inputSchema: { + id: z.string().describe('Provider instance ID to update'), + credentials: z.record(z.string(), z.unknown()).optional().describe('New credentials object (only provide fields you want to update or overwrite)'), + options: z.record(z.string(), z.unknown()).optional().describe('Updated provider options or rate limits'), + enabled: z.boolean().optional().describe('Toggle provider enabled status'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const { id, ...payload } = params; + const result = await client.updateProvider(id, payload); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to update provider '${params.id}'`, error); + } + } + ); + + // 5. delete_provider + server.registerTool( + 'delete_provider', + { + description: 'Delete a provider instance. Fails if the provider is currently referenced as a primary or fallback provider in channel routing.', + inputSchema: { + id: z.string().describe('Provider ID to delete'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.deleteProvider(params.id); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to delete provider '${params.id}'`, error); + } + } + ); + + // 6. test_provider + server.registerTool( + 'test_provider', + { + description: 'Test provider connection and credentials without saving to the database. Can test an existing provider by provider_id or test raw un-persisted credentials.', + inputSchema: { + provider_id: z.string().optional().describe('ID of an existing saved provider to test'), + plugin_name: z.string().optional().describe('Plugin package name to test (required if testing new raw credentials)'), + credentials: z.record(z.string(), z.unknown()).optional().describe('Credentials to test against the provider service'), + options: z.record(z.string(), z.unknown()).optional().describe('Optional parameters to test with'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.testProvider(params); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to test provider connection', error); + } + } + ); + + // 7. get_providers_rate_limits + server.registerTool( + 'get_providers_rate_limits', + { + description: 'Get real-time rate limiter status, token bucket counts, and telemetry across all configured providers in Redis.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getProvidersRateLimits(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to retrieve providers rate limits', error); + } + } + ); + + // 8. get_provider_rate_limit + server.registerTool( + 'get_provider_rate_limit', + { + description: 'Get real-time rate limit status, remaining token capacity, and reset window for a specific provider instance.', + inputSchema: { + id: z.string().describe('Provider instance ID'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getProviderRateLimit(params.id); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to retrieve rate limit for provider '${params.id}'`, error); + } + } + ); + + // 9. reset_provider_rate_limit + server.registerTool( + 'reset_provider_rate_limit', + { + description: 'Manually reset rate limiter tokens and telemetry counters for a specific provider instance in Redis.', + inputSchema: { + id: z.string().describe('Provider instance ID to reset rate limiter for'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.resetProviderRateLimit(params.id); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to reset rate limit for provider '${params.id}'`, error); + } + } + ); +} diff --git a/packages/mcp-server/src/tools/settings.ts b/packages/mcp-server/src/tools/settings.ts new file mode 100644 index 0000000..ae39422 --- /dev/null +++ b/packages/mcp-server/src/tools/settings.ts @@ -0,0 +1,70 @@ +/** + * Tools: Dynamic Operational Configuration Settings + */ + +import { z } from 'zod'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { UserCredentials } from '../auth.js'; +import { ApiClient } from '../api-client.js'; +import { formatApiResponse, formatToolError } from './response.js'; + +export function registerSettingsTools(server: McpServer, getCredentials: () => UserCredentials) { + // 1. get_operational_settings + server.registerTool( + 'get_operational_settings', + { + description: 'Retrieve current dynamic operational settings (worker concurrency, retry limits, delayed and recovery intervals, failure alert thresholds).', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getOperationalSettings(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to get operational settings', error); + } + } + ); + + // 2. update_operational_settings + server.registerTool( + 'update_operational_settings', + { + description: 'Hot-update operational settings across the SimpleNS cluster without requiring service restarts. Changes are broadcast via Redis pub/sub.', + inputSchema: { + settings: z.record(z.string(), z.unknown()).describe('Partial operational settings object to merge and update (e.g. { notification_worker: { batch_size: 20 }, delayed_processor: { poll_interval_ms: 3000 } })'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.updateOperationalSettings(params.settings); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to update operational settings', error); + } + } + ); + + // 3. reset_operational_settings + server.registerTool( + 'reset_operational_settings', + { + description: 'Reset all dynamic operational settings back to initial system defaults.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.resetOperationalSettings(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to reset operational settings', error); + } + } + ); +} diff --git a/packages/mcp-server/tests/verify-stdio.ts b/packages/mcp-server/tests/verify-stdio.ts index ea213f2..2f4c250 100644 --- a/packages/mcp-server/tests/verify-stdio.ts +++ b/packages/mcp-server/tests/verify-stdio.ts @@ -33,8 +33,8 @@ async function main() { console.log(`Found ${tools.tools.length} tools:`); tools.tools.forEach(t => console.log(`- ${t.name}`)); - if (tools.tools.length !== 29) { - throw new Error(`Expected 29 tools, found ${tools.tools.length}`); + if (tools.tools.length !== 54) { + throw new Error(`Expected 54 tools, found ${tools.tools.length}`); } From aa4a6aa9793daaa8527e1d4b1818cb29d271822d Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Fri, 25 Sep 2026 08:18:00 +0530 Subject: [PATCH 29/46] feat: add API key management and usage tracking - Introduced `api_key_id` field in notification schema for tracking. - Enhanced Redis subscriber service with error handling and reconnection logic. - Added API key schemas and types for creation, response, and usage tracking. - Implemented integration tests for API key management, including CRUD operations and usage tracking. - Updated admin authentication routes to use new API key logic. - Refactored auth middleware to support dynamic API keys and admin JWT authentication. - Added unit tests for JWT utilities and API key usage update functions. - Updated notification controller tests to accommodate new API key features. - Enhanced settings API tests to use admin JWT for authorization. - Updated MCP server tests to reflect changes in tool registration. --- .env.example | 9 +- .env.test | 4 +- dashboard/.env.example | 6 +- dashboard/app/api/auth/login/route.ts | 74 +-- dashboard/app/api/auth/logout/route.ts | 17 +- dashboard/app/api/auth/session/route.ts | 24 - dashboard/app/api/auth/setup/route.ts | 26 +- dashboard/app/api/auth/status/route.ts | 5 +- dashboard/app/keys/page.tsx | 544 ++++++++++++++++++ dashboard/app/layout.tsx | 33 +- dashboard/app/setup/page.tsx | 2 +- dashboard/components/auth-provider.tsx | 82 --- .../components/layout/dashboard-layout.tsx | 2 + dashboard/lib/api-client.ts | 30 +- dashboard/lib/api-config.ts | 3 +- dashboard/lib/session.ts | 159 ----- dashboard/lib/types.ts | 41 ++ dashboard/middleware.ts | 206 +------ docker-compose/docker-compose.dev.yaml | 3 +- docker-compose/docker-compose.local.yaml | 3 +- docker-compose/docker-compose.master.yaml | 3 +- package-lock.json | 121 +++- package.json | 2 + scripts/load-test.js | 4 +- src/api/controllers/admin-auth.controller.ts | 60 +- src/api/controllers/api-key.controller.ts | 194 +++++++ .../controllers/notification.controller.ts | 24 +- src/api/middlewares/auth_middleware.ts | 129 ++++- src/api/routes/admin-auth.routes.ts | 23 +- src/api/routes/api-key.routes.ts | 31 + src/api/server.ts | 11 +- src/api/utils/jwt.utils.ts | 56 ++ src/api/utils/utils.ts | 43 ++ src/config/env.config.ts | 2 +- src/database/models/api-key.models.ts | 74 +++ src/database/models/notification.models.ts | 5 + src/plugins/sync/plugin-sync.service.ts | 22 +- src/types/schemas.ts | 46 ++ src/types/types.ts | 25 + tests/integration/api/admin-auth.test.ts | 56 +- tests/integration/api/api-keys.test.ts | 358 ++++++++++++ .../api/middlewares/auth_middleware.test.ts | 272 +++++---- .../api/notification.controller.test.ts | 1 + tests/integration/api/settings.test.ts | 10 +- tests/integration/mcp_server.test.ts | 4 +- tests/unit/api/jwt.utils.test.ts | 43 ++ tests/unit/api/utils.test.ts | 64 +++ 47 files changed, 2238 insertions(+), 718 deletions(-) delete mode 100644 dashboard/app/api/auth/session/route.ts create mode 100644 dashboard/app/keys/page.tsx delete mode 100644 dashboard/components/auth-provider.tsx delete mode 100644 dashboard/lib/session.ts create mode 100644 src/api/controllers/api-key.controller.ts create mode 100644 src/api/routes/api-key.routes.ts create mode 100644 src/api/utils/jwt.utils.ts create mode 100644 src/database/models/api-key.models.ts create mode 100644 tests/integration/api/api-keys.test.ts create mode 100644 tests/unit/api/jwt.utils.test.ts diff --git a/.env.example b/.env.example index a9d680b..c673ec7 100644 --- a/.env.example +++ b/.env.example @@ -12,10 +12,9 @@ # ============================================ # API SERVER # ============================================ -NODE_ENV=production # Set to 'production', 'development', or 'test' -# Generate via: openssl rand -base64 32 -NS_API_KEY= # API secret key for authenticating API requests +NODE_ENV=development # Set to 'production', 'development', or 'test' PORT=3000 # API server port +JWT_SECRET= # Secret key for JWT admin tokens (e.g. openssl rand -base64 32) # ============================================ # DATABASE @@ -44,10 +43,8 @@ LOG_TO_FILE=true # Enable file transport logging to /logs dire # ADMIN DASHBOARD # ============================================ AUTH_TRUST_HOST=true # Trust host header for authentication cookies -AUTH_SECRET= # Dashboard session signing secret (defaults to NS_API_KEY if omitted) API_BASE_URL=http://localhost:3000 # Core Notification service API URL WEBHOOK_HOST=dashboard # Webhook callback host WEBHOOK_PORT=3002 # Dashboard webhook port DASHBOARD_PORT=3002 # Next.js Dashboard port -BASE_PATH=/dashboard # Optional subpath (e.g. /dashboard) or empty for root -HTTPS_COOKIE=false # Controls cookie Secure flag ('true' for HTTPS prod, 'false' for local HTTP) \ No newline at end of file +BASE_PATH=/dashboard # Optional subpath (e.g. /dashboard) or empty for root \ No newline at end of file diff --git a/.env.test b/.env.test index 9ea1e7b..0b90e65 100644 --- a/.env.test +++ b/.env.test @@ -3,6 +3,7 @@ NODE_ENV=test PORT=3099 +JWT_SECRET=test-jwt-secret-key-for-testing-purposes-only # Test Database - uses in-memory MongoDB via mongodb-memory-server # The MONGO_URI below is a placeholder; tests use MongoMemoryServer @@ -14,9 +15,6 @@ REDIS_URL=redis://localhost:6380 # Test Kafka - fully mocked in tests BROKERS=localhost:9093 -# API Authentication -NS_API_KEY=test-api-key-for-testing - # Limits MAX_BATCH_REQ_LIMIT=1000 DELAYED_PARTITION=1 diff --git a/dashboard/.env.example b/dashboard/.env.example index 0ec52af..4fb8724 100644 --- a/dashboard/.env.example +++ b/dashboard/.env.example @@ -1,9 +1,7 @@ +NODE_ENV= # set as production, development, test etc.. used for setting the secure propery for cookie MONGO_URI= # mongodb connection uri PORT=3002 # port number -AUTH_SECRET= # authentication secret (optional: defaults to NS_API_KEY) -NS_API_KEY= # notification service api key -API_BASE_URL= # notification service base url +API_BASE_URL= # notification service base url (e.g. http://localhost:3000) WEBHOOK_HOST= # webhook host WEBHOOK_PORT= # webhook port BASE_PATH= # Base path for Next.js app (e.g., /dashboard). Leave empty for root path. -HTTPS_COOKIE= # Controls the storage of session cookies in http and https mode. (If set false, then the cookies can be stored in http connection) diff --git a/dashboard/app/api/auth/login/route.ts b/dashboard/app/api/auth/login/route.ts index 118e11f..daacb39 100644 --- a/dashboard/app/api/auth/login/route.ts +++ b/dashboard/app/api/auth/login/route.ts @@ -1,8 +1,7 @@ import { NextResponse } from "next/server"; import axios from "axios"; -import { createSession, validateCredentials } from "@/lib/session"; import { getBasePath } from "@/lib/utils"; -import { API_BASE_URL, NS_API_KEY } from "@/lib/api-config"; +import { API_BASE_URL } from "@/lib/api-config"; export async function POST(request: Request) { try { @@ -16,55 +15,42 @@ export async function POST(request: Request) { ); } - let verifiedUser: { id: string; username: string } | null = null; + const response = await axios.post( + `${API_BASE_URL}/api/admin/auth/login`, + { username, password }, + { + headers: { "Content-Type": "application/json" }, + timeout: 5000, + } + ); - // 1. Try verifying via backend Core API (MongoDB credentials) - try { - const response = await axios.post( - `${API_BASE_URL}/api/admin/auth/verify`, - { username, password }, - { - headers: { - Authorization: `Bearer ${NS_API_KEY}`, - "Content-Type": "application/json", - }, - timeout: 5000, - } - ); + const basePath = getBasePath(); + const nextRes = NextResponse.json({ + success: true, + redirectUrl: `${basePath}/dashboard`, + user: response.data?.user, + }); - if (response.data?.isValid && response.data?.user) { - verifiedUser = response.data.user; - } - } catch (apiErr: unknown) { - // If backend returned 401, it's explicitly invalid credentials - if (axios.isAxiosError(apiErr) && apiErr.response?.status === 401) { - return NextResponse.json( - { error: "Invalid username or password" }, - { status: 401 } - ); - } - // Fallback for network issues / local dev without backend running - const fallbackResult = validateCredentials(username, password); - if (fallbackResult.isValid && fallbackResult.userId) { - verifiedUser = { id: fallbackResult.userId, username }; - } + const token = response.data?.token; + if (token) { + nextRes.cookies.set("simplens_session", token, { + httpOnly: true, + sameSite: "lax", + secure: process.env.NODE_ENV === "production", + maxAge: 7 * 24 * 60 * 60, + path: "/", + }); } - if (!verifiedUser) { + return nextRes; + } catch (error: unknown) { + if (axios.isAxiosError(error) && error.response) { + const data = error.response.data as { message?: string; error?: string }; return NextResponse.json( - { error: "Invalid username or password" }, - { status: 401 } + { error: data?.message || data?.error || "Invalid username or password" }, + { status: error.response.status } ); } - - await createSession(verifiedUser.id, verifiedUser.username); - - const basePath = getBasePath(); - return NextResponse.json({ - success: true, - redirectUrl: `${basePath}/dashboard`, - }); - } catch (error) { console.error("Login error:", error); return NextResponse.json( { error: "An error occurred during login" }, diff --git a/dashboard/app/api/auth/logout/route.ts b/dashboard/app/api/auth/logout/route.ts index b018613..c31cc8c 100644 --- a/dashboard/app/api/auth/logout/route.ts +++ b/dashboard/app/api/auth/logout/route.ts @@ -1,11 +1,22 @@ import { NextResponse } from "next/server"; -import { destroySession } from "@/lib/session"; +import { cookies } from "next/headers"; +import axios from "axios"; +import { API_BASE_URL } from "@/lib/api-config"; export async function POST() { try { - await destroySession(); + const cookieStore = await cookies(); + cookieStore.delete("simplens_session"); - return NextResponse.json({ success: true }); + try { + await axios.post(`${API_BASE_URL}/api/admin/auth/logout`, {}, { timeout: 3000 }); + } catch { + // Ignore backend errors on logout + } + + const res = NextResponse.json({ success: true }); + res.cookies.delete("simplens_session"); + return res; } catch (error) { console.error("Logout error:", error); return NextResponse.json( diff --git a/dashboard/app/api/auth/session/route.ts b/dashboard/app/api/auth/session/route.ts deleted file mode 100644 index 7f79a0a..0000000 --- a/dashboard/app/api/auth/session/route.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { NextResponse } from "next/server"; -import { validateSession } from "@/lib/session"; - -export async function GET() { - try { - const session = await validateSession(); - - if (!session.isValid) { - return NextResponse.json({ - authenticated: false, - }); - } - - return NextResponse.json({ - authenticated: true, - user: session.user, - }); - } catch (error) { - console.error("Session check error:", error); - return NextResponse.json({ - authenticated: false, - }); - } -} diff --git a/dashboard/app/api/auth/setup/route.ts b/dashboard/app/api/auth/setup/route.ts index 7ea7002..4a534a5 100644 --- a/dashboard/app/api/auth/setup/route.ts +++ b/dashboard/app/api/auth/setup/route.ts @@ -1,7 +1,6 @@ import { NextResponse } from "next/server"; import axios from "axios"; -import { API_BASE_URL, NS_API_KEY } from "@/lib/api-config"; -import { createSession } from "@/lib/session"; +import { API_BASE_URL } from "@/lib/api-config"; import { getBasePath } from "@/lib/utils"; export async function POST(request: Request) { @@ -32,26 +31,35 @@ export async function POST(request: Request) { } const response = await axios.post( - `${API_BASE_URL}/api/admin/auth/setup`, + `${API_BASE_URL}/api/admin/auth/signup`, { username, password }, { headers: { - Authorization: `Bearer ${NS_API_KEY}`, "Content-Type": "application/json", }, timeout: 5000, } ); - if (response.data.success && response.data.user) { - await createSession(response.data.user.id, response.data.user.username); - } - const basePath = getBasePath(); - return NextResponse.json({ + const nextRes = NextResponse.json({ success: true, redirectUrl: `${basePath}/dashboard`, + user: response.data?.user, }); + + const token = response.data?.token; + if (token) { + nextRes.cookies.set("simplens_session", token, { + httpOnly: true, + sameSite: "lax", + secure: process.env.NODE_ENV === "production", + maxAge: 7 * 24 * 60 * 60, + path: "/", + }); + } + + return nextRes; } catch (error: unknown) { console.error("Admin setup error:", error); if (axios.isAxiosError(error) && error.response?.data) { diff --git a/dashboard/app/api/auth/status/route.ts b/dashboard/app/api/auth/status/route.ts index bad55d4..36dad79 100644 --- a/dashboard/app/api/auth/status/route.ts +++ b/dashboard/app/api/auth/status/route.ts @@ -1,13 +1,10 @@ import { NextResponse } from "next/server"; import axios from "axios"; -import { API_BASE_URL, NS_API_KEY } from "@/lib/api-config"; +import { API_BASE_URL } from "@/lib/api-config"; export async function GET() { try { const response = await axios.get(`${API_BASE_URL}/api/admin/auth/status`, { - headers: { - Authorization: `Bearer ${NS_API_KEY}`, - }, timeout: 5000, }); diff --git a/dashboard/app/keys/page.tsx b/dashboard/app/keys/page.tsx new file mode 100644 index 0000000..e14f426 --- /dev/null +++ b/dashboard/app/keys/page.tsx @@ -0,0 +1,544 @@ +"use client"; + +import { useState } from "react"; +import useSWR from "swr"; +import { DashboardLayout } from "@/components/layout/dashboard-layout"; +import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from "@/components/ui/dialog"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { + Key, + Plus, + Copy, + Check, + Trash2, + Ban, + BarChart3, + MoreVertical, + Activity, + CheckCircle2, +} from "lucide-react"; +import { toast } from "sonner"; +import { apiClient, apiKeyService } from "@/lib/api-client"; +import type { ApiKey, ApiKeyUsageDetailResponse } from "@/lib/types"; + +const fetcher = (url: string): Promise => apiClient.get(url) as unknown as Promise; + +export default function ApiKeysPage() { + const { data, isLoading, mutate } = useSWR<{ keys: ApiKey[] }>("/api/keys", fetcher); + const keys = data?.keys || []; + + // Create Modal state + const [createOpen, setCreateOpen] = useState(false); + const [newKeyName, setNewKeyName] = useState(""); + const [isCreating, setIsCreating] = useState(false); + + // Reveal Key Modal state + const [revealedKey, setRevealedKey] = useState<{ name: string; raw_key: string } | null>(null); + const [copied, setCopied] = useState(false); + + // Action dialogs + const [revokeKeyId, setRevokeKeyId] = useState(null); + const [deleteKeyId, setDeleteKeyId] = useState(null); + + // Usage Details Sheet/Modal state + const [selectedUsage, setSelectedUsage] = useState(null); + + const handleCreateKey = async (e: React.FormEvent) => { + e.preventDefault(); + if (!newKeyName.trim()) { + toast.error("Please enter a key name"); + return; + } + + setIsCreating(true); + try { + const res = await apiKeyService.create({ name: newKeyName.trim() }); + toast.success("API key created successfully"); + setCreateOpen(false); + setNewKeyName(""); + setRevealedKey({ name: res.key.name, raw_key: res.raw_key }); + mutate(); + } catch (err: unknown) { + console.error(err); + toast.error("Failed to create API key"); + } finally { + setIsCreating(false); + } + }; + + const handleCopyKey = () => { + if (!revealedKey) return; + navigator.clipboard.writeText(revealedKey.raw_key); + setCopied(true); + toast.success("API Key copied to clipboard"); + setTimeout(() => setCopied(false), 2000); + }; + + const handleRevoke = async () => { + if (!revokeKeyId) return; + try { + await apiKeyService.revoke(revokeKeyId); + toast.success("API key has been revoked"); + mutate(); + } catch (err) { + console.error(err); + toast.error("Failed to revoke API key"); + } finally { + setRevokeKeyId(null); + } + }; + + const handleDelete = async () => { + if (!deleteKeyId) return; + try { + await apiKeyService.delete(deleteKeyId); + toast.success("API key deleted permanently"); + mutate(); + } catch (err) { + console.error(err); + toast.error("Failed to delete API key"); + } finally { + setDeleteKeyId(null); + } + }; + + const handleViewUsage = async (keyId: string) => { + try { + const detail = await apiKeyService.getUsage(keyId); + setSelectedUsage(detail); + } catch (err) { + console.error(err); + toast.error("Failed to load key usage statistics"); + } + }; + + // Calculate metrics + const totalKeys = keys.length; + const activeKeys = keys.filter((k) => k.status === "active").length; + const totalNotifications = keys.reduce( + (sum, k) => sum + (k.usage?.total_notifications || 0), + 0 + ); + + return ( + +
+ {/* Actions Toolbar */} +
+ +
+ + {/* Metrics Cards */} +
+ + + Total API Keys + + + +
{totalKeys}
+

+ {activeKeys} active, {totalKeys - activeKeys} revoked +

+ + + + + + Total Ingested Notifications + + + +
{totalNotifications.toLocaleString()}
+

+ Processed across all generated API keys +

+ + +
+ + {/* API Keys Table */} + + + Active Credentials + + API keys allow external systems and automation workers to interact with SimpleNS. + + + + {isLoading ? ( +
+ + + +
+ ) : keys.length === 0 ? ( +
+ +

No API keys found

+

Create an API key to allow external services to send notifications.

+
+ ) : ( +
+ + + + Name + Key Prefix + Status + Usage (Notifications) + Channel Breakdown + Last Used + Actions + + + + {keys.map((key) => { + const channels = key.usage?.by_channel || {}; + const channelEntries = Object.entries(channels); + + return ( + + + {key.name} + + + + {key.key_prefix} + + + + {key.status === "active" ? ( + + Active + + ) : ( + + Revoked + + )} + + + + {(key.usage?.total_notifications || 0).toLocaleString()} + + + ({key.usage?.total_requests || 0} reqs) + + + +
+ {channelEntries.length > 0 ? ( + channelEntries.map(([ch, count]) => ( + + {ch}: {count} + + )) + ) : ( + None + )} +
+ + + {key.usage?.last_used_at + ? new Date(key.usage.last_used_at).toLocaleString() + : "Never"} + + + + + + + + handleViewUsage(key.key_id)}> + + View Details & Stats + + {key.status === "active" && ( + setRevokeKeyId(key.key_id)} + > + + Revoke Key + + )} + setDeleteKeyId(key.key_id)} + > + + Delete Key + + + + + + ); + })} + +
+
+ )} + + + + {/* Create Key Modal */} + + +
+ + Create External API Key + + Enter a descriptive name for this API key. The key secret will be displayed once after creation. + + +
+ + setNewKeyName(e.target.value)} + required + autoFocus + /> +
+ + + + +
+ + + + {/* Key Reveal Dialog */} + { + if (!open) setRevealedKey(null); + }} + > + + + + + API Key Created: {revealedKey?.name} + + + Please copy your API key secret immediately. For security reasons, you will not be able to view it again. + + + +
+
+ + +
+ +
+ Keep this key confidential. External applications should pass this key in the Authorization: Bearer <key> header. +
+
+ + + + + + + + {/* Usage Detail Dialog */} + !open && setSelectedUsage(null)}> + + + + + API Key Usage: {selectedUsage?.key.name} + + + Real-time delivery outcomes and volume tracking for this key. + + + + {selectedUsage && ( +
+
+
+
Total Ingested
+
+ {selectedUsage.key.usage?.total_notifications.toLocaleString()} +
+
+
+
Total API Requests
+
+ {selectedUsage.key.usage?.total_requests.toLocaleString()} +
+
+
+ +
+

+ Delivery Status Breakdown +

+
+
+
+ {selectedUsage.status_breakdown.delivered} +
+
Delivered
+
+
+
+ {selectedUsage.status_breakdown.failed} +
+
Failed
+
+
+
+ {selectedUsage.status_breakdown.processing} +
+
Processing
+
+
+
+ {selectedUsage.status_breakdown.pending} +
+
Pending
+
+
+
+ +
+

+ Notifications by Channel +

+
+ {Object.entries(selectedUsage.key.usage?.by_channel || {}).length > 0 ? ( + Object.entries(selectedUsage.key.usage.by_channel).map(([channel, count]) => ( +
+ {channel} + {count.toLocaleString()} +
+ )) + ) : ( +

No channel activity recorded yet.

+ )} +
+
+
+ )} + + + + + + + + {/* Revoke Confirmation Dialog */} + !open && setRevokeKeyId(null)}> + + + Revoke API Key? + + Once revoked, any external service or tool attempting to authenticate with this API key will be immediately blocked with 401 Unauthorized. + + + + Cancel + + Revoke Key + + + + + + {/* Delete Confirmation Dialog */} + !open && setDeleteKeyId(null)}> + + + Delete API Key Permanently? + + This will permanently remove the API key record. This action cannot be undone. + + + + Cancel + + Delete Permanently + + + + +
+ + ); +} diff --git a/dashboard/app/layout.tsx b/dashboard/app/layout.tsx index b53af14..5bffc45 100644 --- a/dashboard/app/layout.tsx +++ b/dashboard/app/layout.tsx @@ -3,7 +3,6 @@ import { Geist, Geist_Mono } from "next/font/google"; import Script from "next/script"; import { Toaster } from "@/components/ui/sonner"; import { ThemeProvider } from "@/components/theme-provider"; -import { AuthProvider } from "@/components/auth-provider"; import { GlassmorphismProvider } from "@/components/glassmorphism-provider"; import { SidebarPinProvider } from "@/components/sidebar-pin-provider"; import { TourProvider } from "@/components/tour/tour-provider"; @@ -38,23 +37,21 @@ export default function RootLayout({ - - - - - - {children} - - - - - - + + + + + {children} + + + + + ); diff --git a/dashboard/app/setup/page.tsx b/dashboard/app/setup/page.tsx index 09f9fd6..ae22d60 100644 --- a/dashboard/app/setup/page.tsx +++ b/dashboard/app/setup/page.tsx @@ -202,7 +202,7 @@ export default function SetupPage() { setIsLoading(true); try { - await authService.setup({ username: trimmedUser, password }); + await authService.signup({ username: trimmedUser, password }); // Check if any plugins are already installed try { diff --git a/dashboard/components/auth-provider.tsx b/dashboard/components/auth-provider.tsx deleted file mode 100644 index f612a4a..0000000 --- a/dashboard/components/auth-provider.tsx +++ /dev/null @@ -1,82 +0,0 @@ -"use client"; - -import { createContext, useContext, useEffect, useState, useCallback } from "react"; -import { useRouter } from "next/navigation"; -import { withBasePath } from "@/lib/utils"; -import { authService } from "@/lib/api-client"; - -interface User { - id: string; - username: string; -} - -interface AuthContextType { - user: User | null; - isAuthenticated: boolean; - isLoading: boolean; - logout: () => Promise; - refreshSession: () => Promise; -} - -const AuthContext = createContext(undefined); - -export function AuthProvider({ children }: { children: React.ReactNode }) { - const router = useRouter(); - const [user, setUser] = useState(null); - const [isLoading, setIsLoading] = useState(true); - - const refreshSession = useCallback(async () => { - try { - const data = await authService.getSession(); - - if (data.authenticated && data.user) { - setUser(data.user); - } else { - setUser(null); - } - } catch (error) { - console.error("Failed to fetch session:", error); - setUser(null); - } finally { - setIsLoading(false); - } - }, []); - - const logout = useCallback(async () => { - try { - await authService.logout(); - setUser(null); - router.push(withBasePath(`/login`)); - router.refresh(); - } catch (error) { - console.error("Logout failed:", error); - window.location.href = withBasePath(`/login`); - } - }, [router]); - - useEffect(() => { - refreshSession(); - }, [refreshSession]); - - return ( - - {children} - - ); -} - -export function useAuth() { - const context = useContext(AuthContext); - if (context === undefined) { - throw new Error("useAuth must be used within an AuthProvider"); - } - return context; -} diff --git a/dashboard/components/layout/dashboard-layout.tsx b/dashboard/components/layout/dashboard-layout.tsx index ece91f5..5f29cce 100644 --- a/dashboard/components/layout/dashboard-layout.tsx +++ b/dashboard/components/layout/dashboard-layout.tsx @@ -17,6 +17,7 @@ import { FileText, Code, BellRing, + Key, } from "lucide-react"; import { Sidebar, @@ -81,6 +82,7 @@ const navCategories: NavCategory[] = [ { label: "System", items: [ + { title: "API Keys", href: "/keys", icon: Key }, { title: "Admin Alerts", href: "/admin-alerts", icon: BellRing }, ], }, diff --git a/dashboard/lib/api-client.ts b/dashboard/lib/api-client.ts index ddd531d..1f443cb 100644 --- a/dashboard/lib/api-client.ts +++ b/dashboard/lib/api-client.ts @@ -29,6 +29,10 @@ import { AdminAuthStatus, ProviderRateLimitStatus, ProviderRateLimitListResponse, + ApiKey, + CreateApiKeyPayload, + CreateApiKeyResponse, + ApiKeyUsageDetailResponse, } from './types'; export class ApiError extends Error { @@ -47,6 +51,7 @@ export const apiClient = axios.create({ headers: { 'Content-Type': 'application/json', }, + withCredentials: true, }); // Dynamic Base URL Resolution @@ -79,15 +84,16 @@ apiClient.interceptors.response.use( // Service Modules export const authService = { - login: (payload: Record): Promise<{ success: boolean; redirectUrl: string }> => + signup: (payload: { username: string; password: string }): Promise<{ success: boolean; message?: string; redirectUrl?: string }> => + apiClient.post('/api/auth/setup', payload), + setup: (payload: { username: string; password: string }): Promise<{ success: boolean; message?: string; redirectUrl?: string }> => + apiClient.post('/api/auth/setup', payload), + login: (payload: Record): Promise<{ success: boolean; isValid?: boolean; message?: string; redirectUrl?: string }> => apiClient.post('/api/auth/login', payload), - logout: (): Promise => apiClient.post('/api/auth/logout'), - getSession: (): Promise<{ authenticated: boolean; user?: { id: string; username: string } }> => - apiClient.get('/api/auth/session'), + logout: (): Promise<{ success: boolean; message?: string }> => + apiClient.post('/api/auth/logout'), getAuthStatus: (): Promise => apiClient.get('/api/auth/status'), - setup: (payload: { username: string; password: string }): Promise<{ success: boolean; redirectUrl: string; message?: string }> => - apiClient.post('/api/auth/setup', payload), }; export const settingsService = { @@ -237,3 +243,15 @@ export const dashboardService = { getTrends: (period?: string): Promise => apiClient.get('/api/dashboard/trends', { params: { period } }), }; + +export const apiKeyService = { + list: (): Promise<{ keys: ApiKey[] }> => apiClient.get('/api/keys'), + create: (payload: CreateApiKeyPayload): Promise => + apiClient.post('/api/keys', payload), + getUsage: (id: string): Promise => + apiClient.get(`/api/keys/${id}`), + revoke: (id: string): Promise<{ success: boolean; message: string; key: ApiKey }> => + apiClient.post(`/api/keys/${id}/revoke`), + delete: (id: string): Promise<{ success: boolean; message: string }> => + apiClient.delete(`/api/keys/${id}`), +}; diff --git a/dashboard/lib/api-config.ts b/dashboard/lib/api-config.ts index c777abd..7fc6641 100644 --- a/dashboard/lib/api-config.ts +++ b/dashboard/lib/api-config.ts @@ -3,5 +3,4 @@ * Centralized configuration for API endpoints */ -export const API_BASE_URL = process.env.API_BASE_URL || 'http://localhost:3000'; -export const NS_API_KEY = process.env.NS_API_KEY || ""; \ No newline at end of file +export const API_BASE_URL = process.env.API_BASE_URL || 'http://localhost:3000'; \ No newline at end of file diff --git a/dashboard/lib/session.ts b/dashboard/lib/session.ts deleted file mode 100644 index d213e9c..0000000 --- a/dashboard/lib/session.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { cookies } from "next/headers"; -import { createHmac } from "crypto"; - -const SESSION_COOKIE_NAME = "simplens_session"; -const SESSION_EXPIRY_DAYS = 7; - -interface SessionPayload { - userId: string; - username: string; - createdAt: number; - expiresAt: number; -} - -/** - * Get the auth secret from environment, with safe deterministic fallbacks - */ -function getAuthSecret(): string { - const secret = process.env.AUTH_SECRET || process.env.JWT_SECRET || process.env.NS_API_KEY; - if (secret && secret.trim().length > 0) { - return secret.trim(); - } - return "simplens-session-auth-secret-fallback-key-32b"; -} - -/** - * Sign a payload using HMAC-SHA256 - */ -function signPayload(payload: string): string { - const hmac = createHmac("sha256", getAuthSecret()); - hmac.update(payload); - return hmac.digest("hex"); -} - -/** - * Verify a signed token and return the payload if valid - */ -function verifyToken(token: string): SessionPayload | null { - try { - const [payloadBase64, signature] = token.split("."); - if (!payloadBase64 || !signature) { - return null; - } - - const expectedSignature = signPayload(payloadBase64); - if (signature !== expectedSignature) { - return null; - } - - const payload = JSON.parse( - Buffer.from(payloadBase64, "base64").toString("utf-8") - ) as SessionPayload; - - // Check if session has expired - if (Date.now() > payload.expiresAt) { - return null; - } - - return payload; - } catch { - return null; - } -} - -/** - * Create a signed session token - */ -function createToken(payload: SessionPayload): string { - const payloadBase64 = Buffer.from(JSON.stringify(payload)).toString("base64"); - const signature = signPayload(payloadBase64); - return `${payloadBase64}.${signature}`; -} - -/** - * Create a session for the authenticated user - */ -export async function createSession(userId: string, username: string): Promise { - const now = Date.now(); - const expiresAt = now + SESSION_EXPIRY_DAYS * 24 * 60 * 60 * 1000; - - const payload: SessionPayload = { - userId, - username, - createdAt: now, - expiresAt, - }; - - const token = createToken(payload); - - const cookieStore = await cookies(); - cookieStore.set(SESSION_COOKIE_NAME, token, { - httpOnly: true, - secure: process.env.HTTPS_COOKIE === "true", - sameSite: "lax", - path: "/", - maxAge: SESSION_EXPIRY_DAYS * 24 * 60 * 60, - }); -} - -/** - * Validate the current session and return user info if valid - */ -export async function validateSession(): Promise<{ - isValid: boolean; - user?: { id: string; username: string }; -}> { - try { - const cookieStore = await cookies(); - const token = cookieStore.get(SESSION_COOKIE_NAME)?.value; - - if (!token) { - return { isValid: false }; - } - - const payload = verifyToken(token); - if (!payload) { - return { isValid: false }; - } - - return { - isValid: true, - user: { - id: payload.userId, - username: payload.username, - }, - }; - } catch { - return { isValid: false }; - } -} - -/** - * Destroy the current session - */ -export async function destroySession(): Promise { - const cookieStore = await cookies(); - cookieStore.delete(SESSION_COOKIE_NAME); -} - -/** - * Validate credentials against environment variables - */ -export function validateCredentials( - username: string, - password: string -): { isValid: boolean; userId?: string } { - const adminUsername = process.env.ADMIN_USERNAME; - const adminPassword = process.env.ADMIN_PASSWORD; - - if (!adminUsername || !adminPassword) { - console.error("ADMIN_USERNAME or ADMIN_PASSWORD not configured in .env"); - return { isValid: false }; - } - - if (username === adminUsername && password === adminPassword) { - return { isValid: true, userId: "admin-1" }; - } - - return { isValid: false }; -} diff --git a/dashboard/lib/types.ts b/dashboard/lib/types.ts index 4a22295..0d435c3 100644 --- a/dashboard/lib/types.ts +++ b/dashboard/lib/types.ts @@ -401,4 +401,45 @@ export interface ProviderRateLimitListResponse { summary: ProviderRateLimitSummary; } +// ============================================================================ +// API KEY MANAGEMENT TYPES +// ============================================================================ + +export interface ApiKeyUsage { + total_requests: number; + total_notifications: number; + by_channel: Record; + last_used_at?: string | null; +} + +export interface ApiKey { + key_id: string; + name: string; + key_prefix: string; + status: 'active' | 'revoked'; + expires_at?: string | null; + usage: ApiKeyUsage; + created_at: string; +} + +export interface CreateApiKeyPayload { + name: string; + expires_at?: string; +} + +export interface CreateApiKeyResponse { + key: ApiKey; + raw_key: string; +} + +export interface ApiKeyUsageDetailResponse { + key: ApiKey; + status_breakdown: { + pending: number; + processing: number; + delivered: number; + failed: number; + }; +} + diff --git a/dashboard/middleware.ts b/dashboard/middleware.ts index 8ca4fc7..a0db2a7 100644 --- a/dashboard/middleware.ts +++ b/dashboard/middleware.ts @@ -1,6 +1,6 @@ import { NextResponse } from "next/server"; import type { NextRequest } from "next/server"; -import { API_BASE_URL, NS_API_KEY } from "./lib/api-config"; +import { API_BASE_URL } from "./lib/api-config"; const basePath = process.env.BASE_PATH || ""; const SESSION_COOKIE_NAME = "simplens_session"; @@ -15,9 +15,11 @@ function isLocalApiRoute(pathname: string): boolean { return localApiPaths.some((p) => pathname.startsWith(p)); } -function proxyToBackend(request: NextRequest, pathname: string) { +function proxyToBackend(request: NextRequest, pathname: string, jwtToken?: string) { const requestHeaders = new Headers(request.headers); - requestHeaders.set("Authorization", `Bearer ${NS_API_KEY}`); + if (jwtToken) { + requestHeaders.set("Authorization", `Bearer ${jwtToken}`); + } const backendUrl = new URL(`${API_BASE_URL}${pathname}${request.nextUrl.search}`); return NextResponse.rewrite(backendUrl, { @@ -31,11 +33,11 @@ function proxyToBackend(request: NextRequest, pathname: string) { const publicRoutes = [ "/login", "/setup", + "/api/admin/auth", "/api/auth/login", "/api/auth/setup", "/api/auth/status", "/api/auth/logout", - "/api/auth/session", "/api/webhook", ]; @@ -50,147 +52,18 @@ const staticPatterns = [ ".ico", ]; -interface SessionPayload { - userId: string; - username: string; - createdAt: number; - expiresAt: number; -} - -function getAuthSecret(): string { - const secret = process.env.AUTH_SECRET || process.env.JWT_SECRET || process.env.NS_API_KEY; - if (secret && secret.trim().length > 0) { - return secret.trim(); - } - return "simplens-session-auth-secret-fallback-key-32b"; -} - -// Web Crypto API compatible HMAC-SHA256 -async function signPayload(payload: string): Promise { - const encoder = new TextEncoder(); - const secretKey = await crypto.subtle.importKey( - "raw", - encoder.encode(getAuthSecret()), - { name: "HMAC", hash: "SHA-256" }, - false, - ["sign"] - ); - const signature = await crypto.subtle.sign( - "HMAC", - secretKey, - encoder.encode(payload) - ); - return Array.from(new Uint8Array(signature)) - .map((b) => b.toString(16).padStart(2, "0")) - .join(""); -} - -async function verifyToken(token: string): Promise { - try { - const [payloadBase64, signature] = token.split("."); - if (!payloadBase64 || !signature) { - return null; - } - - const expectedSignature = await signPayload(payloadBase64); - if (signature !== expectedSignature) { - return null; - } - - const payload = JSON.parse(atob(payloadBase64)) as SessionPayload; - - // Check if session has expired - if (Date.now() > payload.expiresAt) { - return null; - } - - return payload; - } catch { - return null; - } -} - -async function validateSessionFromRequest(request: NextRequest): Promise<{ +function validateSessionFromRequest(request: NextRequest): { isValid: boolean; - user?: { id: string; username: string }; -}> { - try { - const token = request.cookies.get(SESSION_COOKIE_NAME)?.value; - - if (!token) { - return { isValid: false }; - } - - const payload = await verifyToken(token); - if (!payload) { - return { isValid: false }; - } - - return { - isValid: true, - user: { - id: payload.userId, - username: payload.username, - }, - }; - } catch { - return { isValid: false }; - } -} - -/** - * Validate an API key from the Authorization: Bearer header. - * Uses timing-safe comparison via HMAC to prevent timing attacks. - */ -async function validateApiKeyFromRequest(request: NextRequest): Promise<{ isValid: boolean }> { - try { - if (!NS_API_KEY) { - return { isValid: false }; - } - - const authHeader = request.headers.get("authorization"); - if (!authHeader || !authHeader.startsWith("Bearer ")) { - return { isValid: false }; - } - - const providedKey = authHeader.slice(7); // Remove "Bearer " prefix - if (!providedKey) { - return { isValid: false }; - } - - // Timing-safe comparison using HMAC (Edge-compatible via Web Crypto API) - const encoder = new TextEncoder(); - const key = await crypto.subtle.importKey( - "raw", - encoder.encode("api-key-compare"), - { name: "HMAC", hash: "SHA-256" }, - false, - ["sign"] - ); - - const [providedMac, expectedMac] = await Promise.all([ - crypto.subtle.sign("HMAC", key, encoder.encode(providedKey)), - crypto.subtle.sign("HMAC", key, encoder.encode(NS_API_KEY)), - ]); - - const providedArr = new Uint8Array(providedMac); - const expectedArr = new Uint8Array(expectedMac); - - if (providedArr.length !== expectedArr.length) { - return { isValid: false }; - } - - let match = true; - for (let i = 0; i < providedArr.length; i++) { - if (providedArr[i] !== expectedArr[i]) { - match = false; - } - } - - return { isValid: match }; - } catch { + token?: string; +} { + const token = request.cookies.get(SESSION_COOKIE_NAME)?.value; + if (!token || token.trim().length === 0) { return { isValid: false }; } + return { + isValid: true, + token: token.trim(), + }; } function isPublicRoute(pathname: string): boolean { @@ -218,57 +91,46 @@ export async function middleware(request: NextRequest) { } // STEP 1: Strip base path if present and rewrite the request - // This allows Next.js to serve pages at /dashboard/path while routing internally to /path if (basePath && pathname.startsWith(basePath)) { const strippedPath = pathname.slice(basePath.length) || '/'; - // Create URL with stripped path for internal processing const url = request.nextUrl.clone(); url.pathname = strippedPath; - - // Update pathname for subsequent checks pathname = strippedPath; - // Skip static files - rewrite and return immediately if (isStaticFile(pathname)) { return NextResponse.rewrite(url); } - - // For non-static files, we'll rewrite at the end after auth checks - // Continue with auth checks using the stripped pathname } else { - // No base path or doesn't start with base path - // Skip static files if (isStaticFile(pathname)) { return NextResponse.next(); } } // STEP 2: Validate session from request cookies - const session = await validateSessionFromRequest(request); + const session = validateSessionFromRequest(request); // STEP 3: Handle root path after stripping base path - // When user visits /dashboard (with basePath=/dashboard), it becomes / internally - // Redirect authenticated users to the dashboard route, unauthenticated to login if (pathname === "/") { if (session.isValid) { - // Authenticated: redirect to dashboard route return NextResponse.redirect(new URL(`${basePath}/dashboard`, request.url)); } else { - // Not authenticated: redirect to login return NextResponse.redirect(new URL(`${basePath}/login`, request.url)); } } // STEP 4: Check if authenticated user is trying to access login page if (pathname === "/login" && session.isValid) { - // Redirect to dashboard return NextResponse.redirect(new URL(`${basePath}/dashboard`, request.url)); } - // STEP 6: Allow public routes (for non-authenticated users) + // STEP 4.5: Proxy public backend auth routes directly to API server + if (pathname.startsWith("/api/admin/auth")) { + return proxyToBackend(request, pathname); + } + + // STEP 5: Allow public routes if (isPublicRoute(pathname)) { - // If we stripped base path, rewrite the request if (basePath && request.nextUrl.pathname.startsWith(basePath)) { const url = request.nextUrl.clone(); url.pathname = pathname; @@ -277,14 +139,13 @@ export async function middleware(request: NextRequest) { return NextResponse.next(); } - // STEP 7: Protected routes - require authentication + // STEP 6: Protected routes if (!session.isValid) { - // For API routes, fall back to API key (Bearer token) authentication - // This allows programmatic access (e.g. from MCP server) without a session cookie + // For API routes, fall back to Authorization header (API key or Bearer token) + // passed by external services or MCP server directly through the dashboard if (pathname.startsWith("/api/")) { - const apiKeyAuth = await validateApiKeyFromRequest(request); - if (apiKeyAuth.isValid) { - // Valid API key — allow the request through + const authHeader = request.headers.get("authorization"); + if (authHeader && authHeader.startsWith("Bearer ")) { if (isLocalApiRoute(pathname)) { if (basePath && request.nextUrl.pathname.startsWith(basePath)) { const url = request.nextUrl.clone(); @@ -295,7 +156,7 @@ export async function middleware(request: NextRequest) { } return proxyToBackend(request, pathname); } - // Invalid or missing API key on an API route — return 401 JSON instead of redirect + return NextResponse.json( { error: "Unauthorized: valid session or API key required" }, { status: 401 } @@ -308,9 +169,9 @@ export async function middleware(request: NextRequest) { return NextResponse.redirect(loginUrl); } - // STEP 8: Proxy non-local API requests or perform normal next.js routing/basePath stripping + // STEP 7: Proxy non-local API requests attaching the user's Core API JWT token if (pathname.startsWith("/api/") && !isLocalApiRoute(pathname)) { - return proxyToBackend(request, pathname); + return proxyToBackend(request, pathname, session.token); } if (basePath && request.nextUrl.pathname.startsWith(basePath)) { @@ -324,13 +185,6 @@ export async function middleware(request: NextRequest) { export const config = { matcher: [ - /* - * Match all request paths except for the ones starting with: - * - _next/static (static files) - * - _next/image (image optimization files) - * - favicon.ico (favicon file) - * - public files (images, etc.) - */ "/((?!_next/static|_next/image|favicon.ico|.*\\.png$|.*\\.jpg$|.*\\.svg$).*)", ], }; diff --git a/docker-compose/docker-compose.dev.yaml b/docker-compose/docker-compose.dev.yaml index 60d403a..4402c0d 100644 --- a/docker-compose/docker-compose.dev.yaml +++ b/docker-compose/docker-compose.dev.yaml @@ -24,6 +24,7 @@ services: BROKERS: kafka:9093 LOKI_URL: http://loki:3100 LOG_LEVEL: ${LOG_LEVEL:-info} + JWT_SECRET: ${JWT_SECRET:-simplens-default-jwt-secret-key-change-in-production} SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} volumes: - logs-data-dev:/app/logs @@ -164,9 +165,7 @@ services: PORT: ${DASHBOARD_PORT:-3002} API_BASE_URL: http://api:${PORT:-3000} MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 - AUTH_SECRET: ${AUTH_SECRET:-${NS_API_KEY:-change-this-in-production}} AUTH_TRUST_HOST: "true" - HTTPS_COOKIE: ${HTTPS_COOKIE:-false} WEBHOOK_HOST: dashboard WEBHOOK_PORT: "3002" depends_on: diff --git a/docker-compose/docker-compose.local.yaml b/docker-compose/docker-compose.local.yaml index 61b3c48..9d000ab 100644 --- a/docker-compose/docker-compose.local.yaml +++ b/docker-compose/docker-compose.local.yaml @@ -27,6 +27,7 @@ services: BROKERS: kafka:9093 LOKI_URL: http://loki:3100 LOG_LEVEL: ${LOG_LEVEL:-info} + JWT_SECRET: ${JWT_SECRET:-simplens-default-jwt-secret-key-change-in-production} volumes: - logs-data-local:/app/logs command: [ "node", "dist/api/server.js" ] @@ -168,8 +169,6 @@ services: API_BASE_URL: http://api:${PORT:-3000} MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 AUTH_TRUST_HOST: "true" - AUTH_SECRET: ${AUTH_SECRET:-${NS_API_KEY:-simplens-default-session-secret-change-in-production}} - HTTPS_COOKIE: ${HTTPS_COOKIE:-false} WEBHOOK_HOST: dashboard WEBHOOK_PORT: "3002" depends_on: diff --git a/docker-compose/docker-compose.master.yaml b/docker-compose/docker-compose.master.yaml index dc4831c..c7173e0 100644 --- a/docker-compose/docker-compose.master.yaml +++ b/docker-compose/docker-compose.master.yaml @@ -24,6 +24,7 @@ services: BROKERS: kafka:9093 LOKI_URL: http://loki:3100 LOG_LEVEL: ${LOG_LEVEL:-info} + JWT_SECRET: ${JWT_SECRET:-simplens-default-jwt-secret-key-change-in-production} SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} volumes: - logs-data-master:/app/logs @@ -164,9 +165,7 @@ services: PORT: ${DASHBOARD_PORT:-3002} API_BASE_URL: http://api:${PORT:-3000} MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 - AUTH_SECRET: ${AUTH_SECRET:-${NS_API_KEY:-change-this-in-production}} AUTH_TRUST_HOST: "true" - HTTPS_COOKIE: ${HTTPS_COOKIE:-false} WEBHOOK_HOST: dashboard WEBHOOK_PORT: "3002" depends_on: diff --git a/package-lock.json b/package-lock.json index 6d43e14..44716d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "express-rate-limit": "^8.2.1", "helmet": "^8.1.0", "ioredis": "^5.8.2", + "jsonwebtoken": "^9.0.3", "kafkajs": "^2.2.4", "mongoose": "^9.0.0", "uuid": "^13.0.0", @@ -31,6 +32,7 @@ "@types/express-rate-limit": "^5.1.3", "@types/helmet": "^0.0.48", "@types/ioredis": "^4.28.10", + "@types/jsonwebtoken": "^9.0.10", "@types/kafkajs": "^1.8.2", "@types/mongoose": "^5.11.96", "@types/node": "^24.10.1", @@ -2635,6 +2637,17 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/jsonwebtoken": { + "version": "9.0.10", + "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.10.tgz", + "integrity": "sha512-asx5hIG9Qmf/1oStypjanR7iKTv0gXQ1Ov/jfrX6kS/EO0OFni8orbmGCn0672NHR3kXHwpAwR+B368ZGN/2rA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ms": "*", + "@types/node": "*" + } + }, "node_modules/@types/kafkajs": { "version": "1.8.2", "resolved": "https://registry.npmjs.org/@types/kafkajs/-/kafkajs-1.8.2.tgz", @@ -2662,6 +2675,13 @@ "mongoose": "*" } }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "24.10.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", @@ -3644,6 +3664,12 @@ "node": ">= 0.4.0" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/bundle-name": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", @@ -4359,6 +4385,15 @@ "node": ">= 0.4" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/eciesjs": { "version": "0.4.17", "resolved": "https://registry.npmjs.org/eciesjs/-/eciesjs-0.4.17.tgz", @@ -6265,6 +6300,49 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jsonwebtoken": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", + "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", + "license": "MIT", + "dependencies": { + "jws": "^4.0.1", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/kafkajs": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-2.2.4.tgz", @@ -6610,12 +6688,54 @@ "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", "license": "MIT" }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", + "license": "MIT" + }, "node_modules/lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", "license": "MIT" }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", + "license": "MIT" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", + "license": "MIT" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "license": "MIT" + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", + "license": "MIT" + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "license": "MIT" + }, "node_modules/log-symbols": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-6.0.0.tgz", @@ -8300,7 +8420,6 @@ "version": "7.7.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" diff --git a/package.json b/package.json index 765c37e..bf6ac85 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "@types/express-rate-limit": "^5.1.3", "@types/helmet": "^0.0.48", "@types/ioredis": "^4.28.10", + "@types/jsonwebtoken": "^9.0.10", "@types/kafkajs": "^1.8.2", "@types/mongoose": "^5.11.96", "@types/node": "^24.10.1", @@ -74,6 +75,7 @@ "express-rate-limit": "^8.2.1", "helmet": "^8.1.0", "ioredis": "^5.8.2", + "jsonwebtoken": "^9.0.3", "kafkajs": "^2.2.4", "mongoose": "^9.0.0", "uuid": "^13.0.0", diff --git a/scripts/load-test.js b/scripts/load-test.js index 59684e8..3c3ec2f 100644 --- a/scripts/load-test.js +++ b/scripts/load-test.js @@ -17,7 +17,7 @@ * -s, --subject Subject (for email) * -m, --message Message content (for all channels) * -u, --url Base API URL (default: http://localhost:3000) - * -k, --key SimpleNS API Key (defaults to NS_API_KEY env or dev key) + * -k, --key SimpleNS API Key (defaults to API_KEY env or provided key) * -h, --host Webhook host type: 'local', 'docker', or custom hostname (default: local) * -w, --webhook Explicit webhook URL override * -v, --variables Template variables as JSON string (e.g. '{"name":"Alex"}') @@ -65,7 +65,7 @@ const config = { subject: 'Test Notification', message: 'This is a test notification from SimpleNS load test script.', baseUrl: process.env.BASE_URL || 'http://localhost:3000', - apiKey: process.env.NS_API_KEY || '4YCItWcuH2qJe3bXM9LbsbqefflWFlXlzvneMRSSQhU=', + apiKey: process.env.API_KEY || process.env.NS_API_KEY || '', hostType: 'local', webhookUrl: null, variables: null, diff --git a/src/api/controllers/admin-auth.controller.ts b/src/api/controllers/admin-auth.controller.ts index df8bb17..5e3c98a 100644 --- a/src/api/controllers/admin-auth.controller.ts +++ b/src/api/controllers/admin-auth.controller.ts @@ -11,8 +11,21 @@ import system_config_model from '@src/database/models/system-config.models.js'; import { adminSetupSchema, adminLoginSchema } from '@src/types/schemas.js'; import type { AdminCredentialsDoc } from '@src/types/types.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; +import { generateAdminJwt } from '../utils/jwt.utils.js'; export const ADMIN_CREDENTIALS_KEY = 'admin_credentials'; +export const SESSION_COOKIE_NAME = 'simplens_session'; +const SESSION_COOKIE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days + +const setSessionCookie = (res: Response, token: string): void => { + res.cookie(SESSION_COOKIE_NAME, token, { + httpOnly: true, + secure: process.env.NODE_ENV === 'production', + sameSite: 'lax', + path: '/', + maxAge: SESSION_COOKIE_MAX_AGE_MS, + }); +}; /** * Check if the administrator credentials have been configured @@ -74,13 +87,15 @@ export const setupAdmin = async (req: Request, res: Response): Promise => logger.success(`Administrator account configured successfully for username: ${username}`); + const user = { id: 'admin-1', username }; + const token = generateAdminJwt(user); + setSessionCookie(res, token); + res.status(201).json({ success: true, message: 'Administrator account configured successfully.', - user: { - id: 'admin-1', - username, - }, + token, + user, }); } catch (err) { logger.error('Error during administrator setup', err); @@ -122,9 +137,13 @@ export const verifyAdmin = async (req: Request, res: Response): Promise => const computedHashBuf = Buffer.from(computedHash, 'hex'); if (storedHashBuf.length === computedHashBuf.length && timingSafeEqual(storedHashBuf, computedHashBuf)) { + const user = { id: 'admin-1', username: credentials.username }; + const token = generateAdminJwt(user); + setSessionCookie(res, token); res.status(200).json({ isValid: true, - user: { id: 'admin-1', username: credentials.username }, + token, + user, }); return; } @@ -137,9 +156,13 @@ export const verifyAdmin = async (req: Request, res: Response): Promise => const envUser = process.env.ADMIN_USERNAME; const envPass = process.env.ADMIN_PASSWORD; if (envUser && envPass && username === envUser && password === envPass) { + const user = { id: 'admin-1', username: envUser }; + const token = generateAdminJwt(user); + setSessionCookie(res, token); res.status(200).json({ isValid: true, - user: { id: 'admin-1', username: envUser }, + token, + user, }); return; } @@ -150,3 +173,28 @@ export const verifyAdmin = async (req: Request, res: Response): Promise => res.status(500).json({ error: 'Failed to verify administrator credentials' }); } }; + +/** + * Log out administrator and clear session cookie + * POST /api/admin/auth/logout + */ +export const logoutAdmin = async (_req: Request, res: Response): Promise => { + try { + res.clearCookie(SESSION_COOKIE_NAME, { + path: '/', + httpOnly: true, + secure: process.env.NODE_ENV === 'production', + sameSite: 'lax', + }); + res.status(200).json({ + success: true, + message: 'Logged out successfully.', + }); + } catch (err) { + logger.error('Error during logout', err); + res.status(500).json({ error: 'Failed to log out' }); + } +}; + +export const signupAdmin = setupAdmin; +export const loginAdmin = verifyAdmin; diff --git a/src/api/controllers/api-key.controller.ts b/src/api/controllers/api-key.controller.ts new file mode 100644 index 0000000..18f320e --- /dev/null +++ b/src/api/controllers/api-key.controller.ts @@ -0,0 +1,194 @@ +/** + * API Key Management Controller + * + * Handles creation, listing, revocation, deletion, and usage aggregation + * for external service API keys. + */ + +import type { Request, Response } from 'express'; +import crypto from 'crypto'; +import api_key_model from '@src/database/models/api-key.models.js'; +import notification_model from '@src/database/models/notification.models.js'; +import { createApiKeySchema } from '@src/types/schemas.js'; +import type { ApiKeyDoc, ApiKeyResponse } from '@src/types/types.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +/** + * Format an API Key document for API response (strips secret hash) + */ +function toApiKeyResponse(doc: ApiKeyDoc): ApiKeyResponse { + return { + key_id: doc.key_id, + name: doc.name, + key_prefix: doc.key_prefix, + status: doc.status, + expires_at: doc.expires_at || null, + usage: { + total_requests: doc.usage?.total_requests || 0, + total_notifications: doc.usage?.total_notifications || 0, + by_channel: doc.usage?.by_channel instanceof Map + ? Object.fromEntries(doc.usage.by_channel) + : (doc.usage?.by_channel || {}), + last_used_at: doc.usage?.last_used_at || null, + }, + created_at: doc.created_at || new Date(), + }; +} + +/** + * GET /api/keys - List all API keys + */ +export const listApiKeys = async (_req: Request, res: Response): Promise => { + try { + const keys = await api_key_model.find().sort({ created_at: -1 }); + const responseKeys = keys.map((k) => toApiKeyResponse(k.toObject() as ApiKeyDoc)); + res.status(200).json({ keys: responseKeys }); + } catch (err) { + logger.error('Failed to list API keys', err); + res.status(500).json({ error: 'Failed to retrieve API keys' }); + } +}; + +/** + * POST /api/keys - Create a new API key + */ +export const createApiKey = async (req: Request, res: Response): Promise => { + try { + const validation = createApiKeySchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + error: 'Validation failed', + details: validation.error.flatten().fieldErrors, + }); + return; + } + + const { name, expires_at } = validation.data; + + // Generate cryptographically random API key + const rawKey = `sns_live_${crypto.randomBytes(24).toString('base64url')}`; + const key_prefix = `${rawKey.slice(0, 16)}...`; + const key_hash = crypto.createHash('sha256').update(rawKey).digest('hex'); + const key_id = `ak_${crypto.randomUUID()}`; + + const newDoc = await api_key_model.create({ + key_id, + name, + key_prefix, + key_hash, + status: 'active', + expires_at: expires_at || null, + usage: { + total_requests: 0, + total_notifications: 0, + by_channel: {}, + last_used_at: null, + }, + }); + + logger.success(`Created new API key: ${name} (${key_id})`); + + res.status(201).json({ + key: toApiKeyResponse(newDoc.toObject() as ApiKeyDoc), + raw_key: rawKey, + }); + } catch (err) { + logger.error('Failed to create API key', err); + res.status(500).json({ error: 'Failed to create API key' }); + } +}; + +/** + * GET /api/keys/:id - Get single API key details & notification status breakdown + */ +export const getApiKeyUsage = async (req: Request, res: Response): Promise => { + try { + const { id } = req.params; + const key = await api_key_model.findOne({ key_id: id }); + + if (!key) { + res.status(404).json({ error: 'API key not found' }); + return; + } + + // Aggregate delivery outcomes from notifications collection + const statusBreakdown = await notification_model.aggregate([ + { $match: { api_key_id: id } }, + { $group: { _id: '$status', count: { $sum: 1 } } }, + ]); + + const statusCounts: Record = { + pending: 0, + processing: 0, + delivered: 0, + failed: 0, + }; + + for (const item of statusBreakdown) { + if (item._id && typeof item.count === 'number') { + statusCounts[item._id] = item.count; + } + } + + res.status(200).json({ + key: toApiKeyResponse(key.toObject() as ApiKeyDoc), + status_breakdown: statusCounts, + }); + } catch (err) { + logger.error(`Failed to get API key usage for ${req.params.id}`, err); + res.status(500).json({ error: 'Failed to retrieve API key usage' }); + } +}; + +/** + * POST /api/keys/:id/revoke - Revoke an API key + */ +export const revokeApiKey = async (req: Request, res: Response): Promise => { + try { + const { id } = req.params; + const key = await api_key_model.findOneAndUpdate( + { key_id: id }, + { $set: { status: 'revoked' } }, + { new: true } + ); + + if (!key) { + res.status(404).json({ error: 'API key not found' }); + return; + } + + logger.info(`Revoked API key: ${key.name} (${id})`); + res.status(200).json({ + success: true, + message: 'API key has been revoked successfully', + key: toApiKeyResponse(key.toObject() as ApiKeyDoc), + }); + } catch (err) { + logger.error(`Failed to revoke API key ${req.params.id}`, err); + res.status(500).json({ error: 'Failed to revoke API key' }); + } +}; + +/** + * DELETE /api/keys/:id - Permanently delete an API key + */ +export const deleteApiKey = async (req: Request, res: Response): Promise => { + try { + const { id } = req.params; + const result = await api_key_model.deleteOne({ key_id: id }); + + if (result.deletedCount === 0) { + res.status(404).json({ error: 'API key not found' }); + return; + } + + logger.info(`Deleted API key (${id})`); + res.status(200).json({ + success: true, + message: 'API key deleted successfully', + }); + } catch (err) { + logger.error(`Failed to delete API key ${req.params.id}`, err); + res.status(500).json({ error: 'Failed to delete API key' }); + } +}; diff --git a/src/api/controllers/notification.controller.ts b/src/api/controllers/notification.controller.ts index 1302b3e..1237fd2 100644 --- a/src/api/controllers/notification.controller.ts +++ b/src/api/controllers/notification.controller.ts @@ -1,8 +1,16 @@ import type {Request, Response} from 'express'; import { safeValidateNotificationRequest, safeValidateBatchNotificationRequest } from '@src/types/schemas.js'; -import { convert_notification_request_to_notification_schema, convert_batch_notification_schema_to_notification_schema, DuplicateNotificationError, InvalidProviderChannelError, InvalidContentSchemaError, ProviderNotFoundError} from '../utils/utils.js'; +import { + convert_notification_request_to_notification_schema, + convert_batch_notification_schema_to_notification_schema, + DuplicateNotificationError, + InvalidProviderChannelError, + InvalidContentSchemaError, + ProviderNotFoundError, + process_notifications, + updateApiKeyNotificationUsage, +} from '../utils/utils.js'; import { notification } from '@src/types/types.js'; -import { process_notifications } from '@src/api/utils/utils.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; export const notification_controller = async (req: Request, res: Response)=>{ @@ -11,8 +19,12 @@ export const notification_controller = async (req: Request, res: Response)=>{ const result = safeValidateNotificationRequest(data); if(result.success){ const request_data = result.data; - const notifications: notification[] = await convert_notification_request_to_notification_schema(request_data); + const apiKeyId = req.apiKey?.key_id; + const notifications: notification[] = await convert_notification_request_to_notification_schema(request_data, apiKeyId); const response = await process_notifications(notifications); + + updateApiKeyNotificationUsage(req.apiKey, notifications, response?.created_count ?? 0); + res.status(202).json({ message: 'Notifications are being processed', ...response}); } else{ @@ -66,8 +78,12 @@ export const batch_notification_controller = async (req: Request, res: Response) const result = safeValidateBatchNotificationRequest(data); if(result.success){ const request_data = result.data; - const notifications: notification[] = await convert_batch_notification_schema_to_notification_schema(request_data); + const apiKeyId = req.apiKey?.key_id; + const notifications: notification[] = await convert_batch_notification_schema_to_notification_schema(request_data, apiKeyId); const response = await process_notifications(notifications); + + updateApiKeyNotificationUsage(req.apiKey, notifications, response?.created_count ?? 0); + res.status(202).json({ message: 'Notifications are being processed', ...response}); } else{ diff --git a/src/api/middlewares/auth_middleware.ts b/src/api/middlewares/auth_middleware.ts index ba31f95..cf325d8 100644 --- a/src/api/middlewares/auth_middleware.ts +++ b/src/api/middlewares/auth_middleware.ts @@ -1,37 +1,124 @@ +/** + * Core Authentication Middleware + * + * Authenticates API requests using: + * 1. Admin JWT tokens (issued for the Next.js Dashboard). + * 2. Dynamic API Keys (prefixed with 'sns_' and stored in MongoDB). + */ + import type { Request, Response, NextFunction } from 'express'; -import { env } from '@src/config/env.config.js'; import crypto from 'crypto'; +import api_key_model from '@src/database/models/api-key.models.js'; +import { verifyAdminJwt } from '../utils/jwt.utils.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; +import type { ApiKeyDoc, AdminJwtPayload } from '@src/types/types.js'; + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace Express { + interface Request { + user?: AdminJwtPayload; + apiKey?: ApiKeyDoc; + } + } +} -export const auth_middleware = (req: Request, res: Response, next: NextFunction) => { +async function handleDynamicApiKey( + token: string, + req: Request, + res: Response, + next: NextFunction +): Promise { try { - const authHeader = req.headers.authorization; - if (!authHeader || !authHeader.startsWith('Bearer ')) { - return res.status(401).json({ message: "API KEY missing in Authorization Header or 'Bearer' is missing" }); + const tokenHash = crypto.createHash('sha256').update(token).digest('hex'); + const keyDoc = await api_key_model.findOne({ key_hash: tokenHash, status: 'active' }); + + if (!keyDoc) { + res.status(401).json({ message: 'Invalid API KEY' }); + return; } - const api_key = authHeader.split(' ')[1]; - if (!env.NS_API_KEY || typeof env.NS_API_KEY !== 'string' || env.NS_API_KEY.length === 0) { - logger.error('NS_API_KEY is not configured in the environment.'); - return res.status(500).json({ message: 'Server configuration error' }); + if (keyDoc.expires_at && new Date() > new Date(keyDoc.expires_at)) { + res.status(401).json({ message: 'API key has expired' }); + return; } - if (api_key && typeof api_key === 'string') { - const apiKeyBuf = Buffer.from(api_key); - const serverKeyBuf = Buffer.from(env.NS_API_KEY); + const path = req.originalUrl || req.path; + if (path.includes('/api/admin/auth/setup') || path.includes('/api/admin/auth/verify')) { + res.status(403).json({ + error: 'Forbidden: API keys cannot access administrator credential setup or verification routes.', + }); + return; + } + + req.apiKey = keyDoc.toObject() as ApiKeyDoc; - // Only use timingSafeEqual if lengths are equal. - let isValid = false; - if (apiKeyBuf.length === serverKeyBuf.length) { - isValid = crypto.timingSafeEqual(apiKeyBuf, serverKeyBuf); + void api_key_model.updateOne( + { key_id: keyDoc.key_id }, + { + $set: { 'usage.last_used_at': new Date() }, + $inc: { 'usage.total_requests': 1 }, } - if (isValid) { - return next(); + ).catch((err: unknown) => { + logger.warn(`Failed to update usage for API key ${keyDoc.key_id}`, { error: String(err) }); + }); + + next(); + } catch (err) { + logger.error('Error verifying dynamic API key', err); + res.status(500).json({ message: 'Internal Server Error' }); + } +} + +export const auth_middleware = ( + req: Request, + res: Response, + next: NextFunction +): void | Promise => { + try { + const authHeader = req.headers.authorization; + const xApiKeyHeader = req.headers['x-api-key']; + + let token: string | undefined; + + if (authHeader) { + if (!authHeader.startsWith('Bearer ')) { + res.status(401).json({ message: "API KEY missing in Authorization Header or 'Bearer' is missing" }); + return; } + token = authHeader.slice(7).trim(); + } else if (typeof xApiKeyHeader === 'string' && xApiKeyHeader.trim().length > 0) { + token = xApiKeyHeader.trim(); + } else if (req.headers.cookie) { + const match = req.headers.cookie.match(/(?:^|;\s*)simplens_session=([^;]+)/); + if (match) { + token = decodeURIComponent(match[1]).trim(); + } + } + + if (!token) { + res.status(401).json({ message: "API KEY missing in Authorization Header or 'Bearer' is missing" }); + return; } - return res.status(401).json({ message: "Invalid API KEY" }); + + // 1. Dynamic API Key (created via Dashboard / API) + if (token.startsWith('sns_')) { + return handleDynamicApiKey(token, req, res, next); + } + + // 2. Admin JWT (issued for Dashboard session) + const adminPayload = verifyAdminJwt(token); + if (adminPayload) { + req.user = adminPayload; + return next(); + } + + // 3. Invalid token + res.status(401).json({ message: 'Invalid API KEY' }); + return; } catch (err: unknown) { - logger.error(`Error in auth middleware`, err); - return res.status(500).json({ message: "Internal Server Error" }); + logger.error('Error in auth middleware', err); + res.status(500).json({ message: 'Internal Server Error' }); + return; } }; \ No newline at end of file diff --git a/src/api/routes/admin-auth.routes.ts b/src/api/routes/admin-auth.routes.ts index b79d082..b690bef 100644 --- a/src/api/routes/admin-auth.routes.ts +++ b/src/api/routes/admin-auth.routes.ts @@ -3,17 +3,26 @@ */ import { Router } from 'express'; -import { getAuthStatus, setupAdmin, verifyAdmin } from '../controllers/admin-auth.controller.js'; +import { + signupAdmin, + loginAdmin, + logoutAdmin, + getAuthStatus, +} from '../controllers/admin-auth.controller.js'; const router = Router(); -// GET /api/admin/auth/status - Check if admin credentials are configured -router.get('/status', getAuthStatus); +// 1. Signup - Initial administrator account creation (only allowed if no admin exists) +router.post('/signup', signupAdmin); + +// 2. Login - Verify administrator credentials and set session cookie +router.post('/login', loginAdmin); -// POST /api/admin/auth/setup - Initial administrator configuration -router.post('/setup', setupAdmin); +// 3. Logout - Clear session cookie +router.post('/logout', logoutAdmin); + +// Check if admin credentials are configured (used by onboarding / MCP server) +router.get('/status', getAuthStatus); -// POST /api/admin/auth/verify - Verify administrator credentials -router.post('/verify', verifyAdmin); export default router; diff --git a/src/api/routes/api-key.routes.ts b/src/api/routes/api-key.routes.ts new file mode 100644 index 0000000..2250424 --- /dev/null +++ b/src/api/routes/api-key.routes.ts @@ -0,0 +1,31 @@ +/** + * API Key Management Routes + */ + +import { Router } from 'express'; +import { + listApiKeys, + createApiKey, + getApiKeyUsage, + revokeApiKey, + deleteApiKey, +} from '../controllers/api-key.controller.js'; + +const router = Router(); + +// GET /api/keys - List all keys +router.get('/', listApiKeys); + +// POST /api/keys - Create a new key +router.post('/', createApiKey); + +// GET /api/keys/:id - Get usage and details for a key +router.get('/:id', getApiKeyUsage); + +// POST /api/keys/:id/revoke - Revoke a key +router.post('/:id/revoke', revokeApiKey); + +// DELETE /api/keys/:id - Delete a key +router.delete('/:id', deleteApiKey); + +export default router; diff --git a/src/api/server.ts b/src/api/server.ts index 267f707..0159dd5 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -2,6 +2,7 @@ import express from 'express'; import type { Request, Response } from 'express'; import { env } from '@src/config/env.config.js'; import { connectMongoDB } from '@src/config/db.config.js'; +import { connectRedis } from '@src/config/redis.config.js'; import { exit } from 'process'; import notification_router from './routes/notification.routes.js'; import plugins_router from './routes/plugins.routes.js'; @@ -14,6 +15,7 @@ import alerts_router from './routes/alerts.routes.js'; import dashboard_router from './routes/dashboard.routes.js'; import settings_router from './routes/settings.routes.js'; import admin_auth_router from './routes/admin-auth.routes.js'; +import api_keys_router from './routes/api-key.routes.js'; import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { auth_middleware } from './middlewares/auth_middleware.js'; import http from 'http'; @@ -44,7 +46,10 @@ app.use(express.json({ limit: '1mb' })); app.use(helmet()); -app.use(cors({ origin: "*" })); //allows all origins +app.use(cors({ + origin: true, + credentials: true, +})); app.get("/api", (req: Request, res: Response) => { res.json({ @@ -72,6 +77,7 @@ app.use('/api/channels/routing', auth_middleware, channel_routing_router); app.use('/api/admin-channels', auth_middleware, admin_channels_router); app.use('/api/templates', auth_middleware, notification_templates_router); app.use('/api/settings', auth_middleware, settings_router); +app.use('/api/keys', auth_middleware, api_keys_router); app.use('/api/admin/auth', admin_auth_router); const start_server = async () => { @@ -82,7 +88,8 @@ const start_server = async () => { // 0. Initialize Dynamic Configuration & seed defaults into MongoDB await dynamicConfig.initialize(true); - // 1. Initialize Redis subscriber & sync handlers for plugins + // 1. Connect Redis & initialize subscriber for plugins + await connectRedis(); await PluginSyncService.startSubscriber(); NpmAuthService.registerSyncHandlers(); await NpmAuthService.syncNpmrc(); diff --git a/src/api/utils/jwt.utils.ts b/src/api/utils/jwt.utils.ts new file mode 100644 index 0000000..99398fb --- /dev/null +++ b/src/api/utils/jwt.utils.ts @@ -0,0 +1,56 @@ +/** + * Admin JWT Utilities + * + * Handles generation and verification of standard RFC 7519 JSON Web Tokens (HS256) + * for administrator authentication between Dashboard and SimpleNS Core API. + */ + +import jwt from 'jsonwebtoken'; +import { env } from '@src/config/env.config.js'; +import type { AdminJwtPayload } from '@src/types/types.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +const DEFAULT_EXPIRY = '7d'; + +/** + * Generate a signed JWT for the authenticated admin account. + */ +export function generateAdminJwt(admin: { id: string; username: string }): string { + const payload: AdminJwtPayload = { + sub: admin.id, + username: admin.username, + role: 'admin', + }; + + return jwt.sign(payload, env.JWT_SECRET, { + expiresIn: DEFAULT_EXPIRY, + algorithm: 'HS256', + }); +} + +/** + * Verify and decode an Admin JWT. + * Returns decoded payload if valid, null otherwise. + */ +export function verifyAdminJwt(token: string): AdminJwtPayload | null { + try { + const decoded = jwt.verify(token, env.JWT_SECRET, { + algorithms: ['HS256'], + }) as AdminJwtPayload; + + if (decoded && decoded.role === 'admin' && decoded.sub) { + return decoded; + } + + return null; + } catch (err) { + if (err instanceof jwt.TokenExpiredError) { + logger.warn('Admin JWT expired', { expiredAt: err.expiredAt }); + } else if (err instanceof jwt.JsonWebTokenError) { + logger.warn('Invalid Admin JWT signature or format', { message: err.message }); + } else { + logger.error('Unexpected error verifying Admin JWT', err); + } + return null; + } +} diff --git a/src/api/utils/utils.ts b/src/api/utils/utils.ts index 1905b71..1b6c7b2 100644 --- a/src/api/utils/utils.ts +++ b/src/api/utils/utils.ts @@ -14,11 +14,13 @@ import { CORE_TOPICS, delayed_notification_topic, SimpleNSProvider, + ApiKeyDoc, } from "@src/types/types.js"; import { NOTIFICATION_STATUS, OUTBOX_STATUS } from "@src/types/types.js"; import mongoose from "mongoose"; import notification_model from "@src/database/models/notification.models.js"; import outbox_model from "@src/database/models/outbox.models.js"; +import api_key_model from "@src/database/models/api-key.models.js"; import { apiLogger as logger } from "@src/workers/utils/logger.js"; import { PluginRegistry } from "@src/plugins/index.js"; import notification_template_model from "@src/database/models/notification-template.models.js"; @@ -60,6 +62,7 @@ const preloadTemplatesByIds = async ( */ export const convert_notification_request_to_notification_schema = async ( data: notification_request, + apiKeyId?: string, ): Promise => { const notifications: notification[] = []; @@ -128,6 +131,7 @@ export const convert_notification_request_to_notification_schema = async ( status: NOTIFICATION_STATUS.pending, scheduled_at: data.scheduled_at, retry_count: 0, + api_key_id: apiKeyId, created_at: new Date(), }; @@ -142,6 +146,7 @@ export const convert_notification_request_to_notification_schema = async ( */ export const convert_batch_notification_schema_to_notification_schema = async ( data: batch_notification_request, + apiKeyId?: string, ): Promise => { const notifications: notification[] = []; @@ -213,6 +218,7 @@ export const convert_batch_notification_schema_to_notification_schema = async ( status: NOTIFICATION_STATUS.pending, scheduled_at: data.scheduled_at, retry_count: 0, + api_key_id: apiKeyId, created_at: new Date(), }; notifications.push(notification_obj); @@ -548,3 +554,40 @@ export const validateContentSchema = ( ); } }; + +/** + * Update notification usage counters for an API key asynchronously. + */ +export const updateApiKeyNotificationUsage = ( + apiKey: ApiKeyDoc | undefined, + notifications: notification[], + createdCount: number, +): void => { + if (!apiKey || createdCount <= 0) { + return; + } + + const channelCounts: Record = {}; + for (const n of notifications) { + channelCounts[n.channel] = (channelCounts[n.channel] || 0) + 1; + } + + const incObj: Record = { + 'usage.total_notifications': createdCount, + }; + for (const [ch, count] of Object.entries(channelCounts)) { + incObj[`usage.by_channel.${ch}`] = count; + } + + void api_key_model.updateOne( + { key_id: apiKey.key_id }, + { + $set: { 'usage.last_used_at': new Date() }, + $inc: incObj, + }, + ).catch((err: unknown) => + logger.warn('Failed to update API key notification usage', { + error: String(err), + }), + ); +}; diff --git a/src/config/env.config.ts b/src/config/env.config.ts index 4063d13..9293b27 100644 --- a/src/config/env.config.ts +++ b/src/config/env.config.ts @@ -23,7 +23,7 @@ export const env = { // API Server PORT: parseInt(process.env.PORT || "3000"), - NS_API_KEY: (process.env.NS_API_KEY || ""), + JWT_SECRET: (process.env.JWT_SECRET || "simplens-default-jwt-secret-key-change-in-production"), DELAYED_PARTITION: parseInt(process.env.DELAYED_PARTITION || "1"), NOTIFICATION_STATUS_PARTITION: parseInt(process.env.NOTIFICATION_STATUS_PARTITION || "1"), diff --git a/src/database/models/api-key.models.ts b/src/database/models/api-key.models.ts new file mode 100644 index 0000000..f0dc1c7 --- /dev/null +++ b/src/database/models/api-key.models.ts @@ -0,0 +1,74 @@ +/** + * API Key Model + * + * Stores hashed API keys and real-time usage metrics for external service authentication. + */ + +import { type ApiKeyDoc } from '@src/types/types.js'; +import mongoose from 'mongoose'; + +const api_key_schema = new mongoose.Schema( + { + key_id: { + type: String, + required: true, + unique: true, + index: true, + }, + name: { + type: String, + required: true, + }, + key_prefix: { + type: String, + required: true, + }, + key_hash: { + type: String, + required: true, + unique: true, + index: true, + }, + status: { + type: String, + enum: ['active', 'revoked'], + default: 'active', + index: true, + }, + expires_at: { + type: Date, + default: null, + }, + usage: { + total_requests: { + type: Number, + default: 0, + }, + total_notifications: { + type: Number, + default: 0, + }, + by_channel: { + type: Map, + of: Number, + default: {}, + }, + last_used_at: { + type: Date, + default: null, + }, + }, + }, + { + timestamps: { + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + } +); + +const api_key_model = + mongoose.models?.ApiKey || + mongoose.model('ApiKey', api_key_schema); + +export default api_key_model; diff --git a/src/database/models/notification.models.ts b/src/database/models/notification.models.ts index 0c4ac1c..e03476f 100644 --- a/src/database/models/notification.models.ts +++ b/src/database/models/notification.models.ts @@ -86,6 +86,11 @@ const notification_schema = new mongoose.Schema( type: Number, default: 0, }, + api_key_id: { + type: String, + default: null, + index: true, + }, // Recovery claiming fields for horizontal scalability recovery_claimed_by: { type: String, diff --git a/src/plugins/sync/plugin-sync.service.ts b/src/plugins/sync/plugin-sync.service.ts index 97f5ae3..70d41fa 100644 --- a/src/plugins/sync/plugin-sync.service.ts +++ b/src/plugins/sync/plugin-sync.service.ts @@ -7,7 +7,7 @@ import { randomUUID } from 'crypto'; import type { Redis as RedisType } from 'ioredis'; -import { getRedisClient } from '@src/config/redis.config.js'; +import { getRedisClient, connectRedis } from '@src/config/redis.config.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; import type { PluginSyncAction, PluginSyncHandler, PluginSyncMessage, PluginSyncPayload } from '@src/types/types.js'; @@ -92,10 +92,25 @@ export class PluginSyncServiceClass { } try { + await connectRedis(); const baseClient = getRedisClient(); this.subClient = baseClient.duplicate(); - await this.subClient.connect(); + this.subClient.on('error', (err: Error) => { + logger.error('Plugin sync Redis subscriber error:', err); + }); + + this.subClient.on('close', () => { + this.isSubscribed = false; + logger.warn('Plugin sync Redis subscriber connection closed. Reconnecting in 3s...'); + setTimeout(() => { + void this.startSubscriber(); + }, 3000); + }); + + if (this.subClient.status !== 'ready' && this.subClient.status !== 'connecting') { + await this.subClient.connect(); + } await this.subClient.subscribe(PLUGIN_SYNC_CHANNEL); this.isSubscribed = true; @@ -140,6 +155,9 @@ export class PluginSyncServiceClass { logger.success(`Subscribed to plugin sync channel: ${PLUGIN_SYNC_CHANNEL}`); } catch (err) { logger.error('Failed to initialize Redis subscriber for plugin sync:', err); + setTimeout(() => { + void this.startSubscriber(); + }, 3000); } } diff --git a/src/types/schemas.ts b/src/types/schemas.ts index 8d5dcc1..49cda84 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -145,6 +145,7 @@ export const notificationSchema = z.object({ status: z.enum(NOTIFICATION_STATUS), error_message: z.string().optional(), retry_count: z.number().int().min(0), + api_key_id: z.string().nullable().optional(), // Recovery claiming fields for horizontal scalability recovery_claimed_by: z.string().nullable().optional(), recovery_claimed_at: z.coerce.date().nullable().optional(), @@ -680,3 +681,48 @@ export const providerRateLimitListResponseSchema = z.object({ summary: providerRateLimitSummarySchema, }); +// ============================================================================ +// API KEY & ADMIN AUTH SCHEMAS +// ============================================================================ + +export const apiKeyUsageSchema = z.object({ + total_requests: z.number().int().min(0).default(0), + total_notifications: z.number().int().min(0).default(0), + by_channel: z.record(z.string(), z.number().int().min(0)).default({}), + last_used_at: z.coerce.date().nullable().optional(), +}); + +export const apiKeyStatusSchema = z.enum(['active', 'revoked']); + +export const apiKeyDocSchema = z.object({ + key_id: z.string(), + name: z.string(), + key_prefix: z.string(), + key_hash: z.string(), + status: apiKeyStatusSchema, + expires_at: z.coerce.date().nullable().optional(), + usage: apiKeyUsageSchema, + created_at: z.coerce.date().optional(), + updated_at: z.coerce.date().optional(), +}); + +export const createApiKeySchema = z.object({ + name: z.string().min(1, 'API key name is required').max(100), + expires_at: z.coerce.date().optional(), +}); + +export const apiKeyResponseSchema = z.object({ + key_id: z.string(), + name: z.string(), + key_prefix: z.string(), + status: apiKeyStatusSchema, + expires_at: z.coerce.date().nullable().optional(), + usage: apiKeyUsageSchema, + created_at: z.coerce.date(), +}); + +export const createApiKeyResponseSchema = z.object({ + key: apiKeyResponseSchema, + raw_key: z.string(), +}); + diff --git a/src/types/types.ts b/src/types/types.ts index 7a9b942..7cd473d 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -145,6 +145,12 @@ import { providerRateLimitStatusSchema, providerRateLimitSummarySchema, providerRateLimitListResponseSchema, + apiKeyUsageSchema, + apiKeyStatusSchema, + apiKeyDocSchema, + createApiKeySchema, + apiKeyResponseSchema, + createApiKeyResponseSchema, } from "./schemas.js"; // ============================================================================ @@ -560,3 +566,22 @@ export type ProviderRateLimitStatus = z.infer; export type ProviderRateLimitListResponse = z.infer; +// ============================================================================ +// API KEY & ADMIN AUTH TYPES +// ============================================================================ + +export interface AdminJwtPayload { + sub: string; + username: string; + role: 'admin'; + iat?: number; + exp?: number; +} + +export type ApiKeyStatus = z.infer; +export type ApiKeyUsage = z.infer; +export type ApiKeyDoc = z.infer; +export type CreateApiKeyInput = z.infer; +export type ApiKeyResponse = z.infer; +export type CreateApiKeyResponse = z.infer; + diff --git a/tests/integration/api/admin-auth.test.ts b/tests/integration/api/admin-auth.test.ts index 7187e26..569638c 100644 --- a/tests/integration/api/admin-auth.test.ts +++ b/tests/integration/api/admin-auth.test.ts @@ -45,10 +45,10 @@ describe('Admin Auth Routes Integration', () => { }); }); - describe('POST /api/admin/auth/setup', () => { + describe('POST /api/admin/auth/signup', () => { it('should reject setup with validation error if password too short', async () => { const res = await request(app) - .post('/api/admin/auth/setup') + .post('/api/admin/auth/signup') .send({ username: 'admin', password: '123' }); expect(res.status).toBe(400); @@ -57,7 +57,7 @@ describe('Admin Auth Routes Integration', () => { it('should reject setup with validation error if password exceeds max length', async () => { const res = await request(app) - .post('/api/admin/auth/setup') + .post('/api/admin/auth/signup') .send({ username: 'admin', password: 'a'.repeat(65) }); expect(res.status).toBe(400); @@ -71,7 +71,7 @@ describe('Admin Auth Routes Integration', () => { } as any); const res = await request(app) - .post('/api/admin/auth/setup') + .post('/api/admin/auth/signup') .send({ username: 'admin', password: 'supersecretpassword123' }); expect(res.status).toBe(400); @@ -83,26 +83,68 @@ describe('Admin Auth Routes Integration', () => { vi.mocked(system_config_model.create).mockResolvedValue({} as any); const res = await request(app) - .post('/api/admin/auth/setup') + .post('/api/admin/auth/signup') .send({ username: 'superuser', password: 'supersecretpassword123' }); expect(res.status).toBe(201); expect(res.body.success).toBe(true); + expect(res.body.token).toBeDefined(); + expect(typeof res.body.token).toBe('string'); expect(res.body.user.username).toBe('superuser'); expect(system_config_model.create).toHaveBeenCalled(); + expect(res.headers['set-cookie']).toBeDefined(); + expect(res.headers['set-cookie'].some((c: string) => c.includes('simplens_session='))).toBe(true); }); }); - describe('POST /api/admin/auth/verify', () => { + describe('POST /api/admin/auth/login', () => { it('should reject invalid credentials', async () => { vi.mocked(system_config_model.findOne).mockResolvedValue(null); const res = await request(app) - .post('/api/admin/auth/verify') + .post('/api/admin/auth/login') .send({ username: 'wrong', password: 'wrongpassword' }); expect(res.status).toBe(401); expect(res.body.isValid).toBe(false); }); + + it('should verify correct credentials and return token', async () => { + const crypto = await import('crypto'); + const salt = crypto.randomBytes(32).toString('hex'); + const password_hash = crypto.scryptSync('supersecretpassword123', salt, 64).toString('hex'); + + vi.mocked(system_config_model.findOne).mockResolvedValue({ + key: 'admin_credentials', + value: { + username: 'admin', + password_hash, + salt, + created_at: new Date().toISOString(), + }, + } as any); + + const res = await request(app) + .post('/api/admin/auth/login') + .send({ username: 'admin', password: 'supersecretpassword123' }); + + expect(res.status).toBe(200); + expect(res.body.isValid).toBe(true); + expect(res.body.token).toBeDefined(); + expect(typeof res.body.token).toBe('string'); + expect(res.body.user.username).toBe('admin'); + expect(res.headers['set-cookie']).toBeDefined(); + expect(res.headers['set-cookie'].some((c: string) => c.includes('simplens_session='))).toBe(true); + }); + }); + + describe('POST /api/admin/auth/logout', () => { + it('should clear session cookie and return 200', async () => { + const res = await request(app).post('/api/admin/auth/logout'); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.headers['set-cookie']).toBeDefined(); + }); }); }); diff --git a/tests/integration/api/api-keys.test.ts b/tests/integration/api/api-keys.test.ts new file mode 100644 index 0000000..010963b --- /dev/null +++ b/tests/integration/api/api-keys.test.ts @@ -0,0 +1,358 @@ +/** + * Integration tests for API Key Management and Usage Tracking + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import crypto from 'crypto'; +import api_keys_router from '@src/api/routes/api-key.routes.js'; +import notification_router from '@src/api/routes/notification.routes.js'; +import admin_auth_router from '@src/api/routes/admin-auth.routes.js'; +import { auth_middleware } from '@src/api/middlewares/auth_middleware.js'; +import api_key_model from '@src/database/models/api-key.models.js'; +import notification_model from '@src/database/models/notification.models.js'; +import system_config_model from '@src/database/models/system-config.models.js'; +import { generateAdminJwt } from '@src/api/utils/jwt.utils.js'; + +// Mock models +vi.mock('@src/database/models/api-key.models.js', () => { + let mockKeys: any[] = []; + return { + default: { + find: vi.fn(() => ({ + sort: vi.fn().mockImplementation(() => Promise.resolve(mockKeys)), + })), + findOne: vi.fn((query: any) => { + if (query.key_hash) { + const match = mockKeys.find(k => k.key_hash === query.key_hash && (!query.status || k.status === query.status)); + return Promise.resolve(match ? { ...match, toObject: () => ({ ...match }) } : null); + } + if (query.key_id) { + const match = mockKeys.find(k => k.key_id === query.key_id); + return Promise.resolve(match ? { ...match, toObject: () => ({ ...match }) } : null); + } + return Promise.resolve(null); + }), + create: vi.fn((data: any) => { + const doc = { + ...data, + created_at: new Date(), + updated_at: new Date(), + toObject: () => ({ ...data, created_at: new Date(), updated_at: new Date() }), + }; + mockKeys.push(doc); + return Promise.resolve(doc); + }), + findOneAndUpdate: vi.fn((query: any, update: any) => { + const match = mockKeys.find(k => k.key_id === query.key_id); + if (!match) return Promise.resolve(null); + if (update.$set) { + Object.assign(match, update.$set); + } + return Promise.resolve({ ...match, toObject: () => ({ ...match }) }); + }), + updateOne: vi.fn((query: any, update: any) => { + const match = mockKeys.find(k => k.key_id === query.key_id); + if (match) { + if (update.$inc) { + for (const [field, val] of Object.entries(update.$inc)) { + if (field.startsWith('usage.by_channel.')) { + const ch = field.replace('usage.by_channel.', ''); + match.usage.by_channel[ch] = (match.usage.by_channel[ch] || 0) + (val as number); + } else if (field === 'usage.total_notifications') { + match.usage.total_notifications = (match.usage.total_notifications || 0) + (val as number); + } else if (field === 'usage.total_requests') { + match.usage.total_requests = (match.usage.total_requests || 0) + (val as number); + } + } + } + if (update.$set) { + Object.assign(match, update.$set); + } + } + return Promise.resolve({ modifiedCount: match ? 1 : 0 }); + }), + deleteOne: vi.fn((query: any) => { + const idx = mockKeys.findIndex(k => k.key_id === query.key_id); + if (idx >= 0) { + mockKeys.splice(idx, 1); + return Promise.resolve({ deletedCount: 1 }); + } + return Promise.resolve({ deletedCount: 0 }); + }), + __resetMockKeys: () => { + mockKeys = []; + }, + __getMockKeys: () => mockKeys, + }, + }; +}); + +vi.mock('@src/database/models/notification.models.js', () => ({ + default: { + aggregate: vi.fn().mockResolvedValue([ + { _id: 'delivered', count: 10 }, + { _id: 'failed', count: 2 }, + ]), + findOne: vi.fn().mockResolvedValue(null), + insertMany: vi.fn().mockResolvedValue([]), + }, +})); + +vi.mock('@src/database/models/outbox.models.js', () => ({ + default: { + insertMany: vi.fn().mockResolvedValue([]), + }, +})); + +vi.mock('@src/database/models/system-config.models.js', () => ({ + default: { + findOne: vi.fn().mockResolvedValue({ key: 'admin_credentials', value: { username: 'admin' } }), + }, +})); + +vi.mock('@src/plugins/index.js', () => { + const mockProviderInstance = { + getContentSchema: () => ({ + safeParse: () => ({ success: true, data: {} }), + }), + }; + + return { + PluginRegistry: { + getProvider: vi.fn().mockReturnValue(mockProviderInstance), + get: vi.fn().mockReturnValue(mockProviderInstance), + getDefaultProvider: vi.fn().mockReturnValue(mockProviderInstance), + hasChannel: vi.fn().mockReturnValue(true), + getChannelProviders: vi.fn().mockReturnValue(['mock-provider']), + }, + }; +}); + +vi.mock('@src/api/utils/utils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + process_notifications: vi.fn().mockResolvedValue({ + created_count: 1, + notification_ids: ['507f1f77bcf86cd799439011'], + duplicate_count: 0, + }), + }; +}); + +describe('API Keys Integration & Usage Tracking', () => { + let app: express.Application; + let adminToken: string; + + beforeEach(() => { + vi.clearAllMocks(); + (api_key_model as any).__resetMockKeys(); + + adminToken = generateAdminJwt({ id: 'admin-1', username: 'admin' }); + + app = express(); + app.use(express.json()); + app.use('/api/keys', auth_middleware, api_keys_router); + app.use('/api/notification', auth_middleware, notification_router); + app.use('/api/admin/auth', auth_middleware, admin_auth_router); + }); + + describe('API Key Management CRUD', () => { + it('should reject unauthenticated access to /api/keys', async () => { + const res = await request(app).get('/api/keys'); + expect(res.status).toBe(401); + }); + + it('should create an API key when authenticated with Admin JWT', async () => { + const res = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Payment Webhooks' }); + + expect(res.status).toBe(201); + expect(res.body.key).toBeDefined(); + expect(res.body.key.name).toBe('Payment Webhooks'); + expect(res.body.key.key_prefix).toMatch(/^sns_live_/); + expect(res.body.raw_key).toMatch(/^sns_live_/); + expect(res.body.key.status).toBe('active'); + }); + + it('should list API keys without exposing key_hash or raw key', async () => { + await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Service Alpha' }); + + const res = await request(app) + .get('/api/keys') + .set('Authorization', `Bearer ${adminToken}`); + + expect(res.status).toBe(200); + expect(res.body.keys).toHaveLength(1); + expect(res.body.keys[0].name).toBe('Service Alpha'); + expect(res.body.keys[0].key_hash).toBeUndefined(); + expect(res.body.keys[0].raw_key).toBeUndefined(); + expect(res.body.keys[0].key_prefix).toBeDefined(); + }); + + it('should get API key usage and delivery breakdown', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Analytics Key' }); + + const keyId = createRes.body.key.key_id; + + const res = await request(app) + .get(`/api/keys/${keyId}`) + .set('Authorization', `Bearer ${adminToken}`); + + expect(res.status).toBe(200); + expect(res.body.key.key_id).toBe(keyId); + expect(res.body.status_breakdown).toEqual({ + delivered: 10, + failed: 2, + pending: 0, + processing: 0, + }); + }); + + it('should revoke an API key', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Temporary Worker' }); + + const keyId = createRes.body.key.key_id; + + const revokeRes = await request(app) + .post(`/api/keys/${keyId}/revoke`) + .set('Authorization', `Bearer ${adminToken}`); + + expect(revokeRes.status).toBe(200); + expect(revokeRes.body.success).toBe(true); + expect(revokeRes.body.key.status).toBe('revoked'); + }); + + it('should delete an API key', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Obsolete Key' }); + + const keyId = createRes.body.key.key_id; + + const deleteRes = await request(app) + .delete(`/api/keys/${keyId}`) + .set('Authorization', `Bearer ${adminToken}`); + + expect(deleteRes.status).toBe(200); + expect(deleteRes.body.success).toBe(true); + + const getRes = await request(app) + .get(`/api/keys/${keyId}`) + .set('Authorization', `Bearer ${adminToken}`); + expect(getRes.status).toBe(404); + }); + }); + + describe('External Services Authentication & Usage Tracking', () => { + it('should authenticate external notification request via generated API key', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Order Notification Service' }); + + const rawKey = createRes.body.raw_key; + expect(rawKey).toBeDefined(); + + // Send notification using the generated API key + const notifPayload = { + request_id: crypto.randomUUID(), + client_id: crypto.randomUUID(), + channel: ['email'], + recipient: { email: 'customer@example.com' }, + content: { email: { subject: 'Receipt', message: 'Order paid' } }, + webhook_url: 'https://webhook.site/test', + }; + + const notifRes = await request(app) + .post('/api/notification') + .set('Authorization', `Bearer ${rawKey}`) + .send(notifPayload); + + expect(notifRes.status).toBe(202); + + // Verify usage tracking incremented + const keys = (api_key_model as any).__getMockKeys(); + const usedKey = keys[0]; + expect(usedKey.usage.total_notifications).toBeGreaterThanOrEqual(1); + expect(usedKey.usage.by_channel['email']).toBeGreaterThanOrEqual(1); + }); + + it('should reject requests made with a revoked API key', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'To Be Revoked' }); + + const keyId = createRes.body.key.key_id; + const rawKey = createRes.body.raw_key; + + // Revoke it + await request(app) + .post(`/api/keys/${keyId}/revoke`) + .set('Authorization', `Bearer ${adminToken}`); + + // Attempt to send notification + const notifRes = await request(app) + .post('/api/notification') + .set('Authorization', `Bearer ${rawKey}`) + .send({ + request_id: crypto.randomUUID(), + client_id: crypto.randomUUID(), + channel: ['email'], + recipient: { email: 'customer@example.com' }, + content: { email: { subject: 'Receipt', message: 'Order paid' } }, + webhook_url: 'https://webhook.site/test', + }); + + expect(notifRes.status).toBe(401); + }); + + it('should forbid API keys from accessing admin credential setup or verify routes', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'External Key' }); + + const rawKey = createRes.body.raw_key; + + const setupRes = await request(app) + .post('/api/admin/auth/setup') + .set('Authorization', `Bearer ${rawKey}`) + .send({ username: 'hacker', password: 'password123' }); + + expect(setupRes.status).toBe(403); + expect(setupRes.body.error).toContain('cannot access administrator credential'); + }); + + it('should allow API keys to access /api/admin/auth/status for MCP server compatibility', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'MCP Server Key' }); + + const rawKey = createRes.body.raw_key; + + const statusRes = await request(app) + .get('/api/admin/auth/status') + .set('Authorization', `Bearer ${rawKey}`); + + expect(statusRes.status).toBe(200); + expect(statusRes.body.isConfigured).toBe(true); + }); + }); +}); diff --git a/tests/integration/api/middlewares/auth_middleware.test.ts b/tests/integration/api/middlewares/auth_middleware.test.ts index f5c5de9..44dccd1 100644 --- a/tests/integration/api/middlewares/auth_middleware.test.ts +++ b/tests/integration/api/middlewares/auth_middleware.test.ts @@ -1,20 +1,23 @@ /** * Integration tests for auth_middleware.ts - * Tests API authentication middleware with mocked environment + * Tests API authentication middleware with Admin JWT and Dynamic API Keys */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import type { Request, Response, NextFunction } from 'express'; +import crypto from 'crypto'; +import { generateAdminJwt } from '../../../../src/api/utils/jwt.utils.js'; -// Mocked env that can be modified per test -let mockApiKey: string | undefined = 'valid-api-key-12345'; +// Mock DB model for api_key_model +const mockFindOne = vi.fn(); +const mockUpdateOne = vi.fn().mockReturnValue({ + catch: vi.fn(), +}); -// Mock env config - using a factory function to return fresh mock -vi.mock('@src/config/env.config.js', () => ({ - env: { - get NS_API_KEY() { - return mockApiKey; - }, +vi.mock('@src/database/models/api-key.models.js', () => ({ + default: { + findOne: (...args: unknown[]) => mockFindOne(...args), + updateOne: (...args: unknown[]) => mockUpdateOne(...args), }, })); @@ -31,12 +34,19 @@ vi.mock('@src/workers/utils/logger.js', () => ({ describe('Auth Middleware', () => { let authMiddleware: typeof import('../../../../src/api/middlewares/auth_middleware.js'); - // Helper to create mock request - const createMockRequest = (authHeader?: string): Partial => ({ - headers: authHeader ? { authorization: authHeader } : {}, + const createMockRequest = ( + authHeader?: string, + path: string = '/api/notifications', + cookieHeader?: string, + ): Partial => ({ + headers: { + ...(authHeader ? { authorization: authHeader } : {}), + ...(cookieHeader ? { cookie: cookieHeader } : {}), + }, + originalUrl: path, + path, }); - // Helper to create mock response const createMockResponse = (): Partial => { const res: Partial = { status: vi.fn().mockReturnThis(), @@ -47,11 +57,6 @@ describe('Auth Middleware', () => { beforeEach(async () => { vi.clearAllMocks(); - vi.resetModules(); - - // Reset to valid API key by default - mockApiKey = 'valid-api-key-12345'; - authMiddleware = await import('../../../../src/api/middlewares/auth_middleware.js'); }); @@ -59,158 +64,201 @@ describe('Auth Middleware', () => { vi.clearAllMocks(); }); - describe('Valid authentication', () => { - it('should call next() for valid API key', () => { - const req = createMockRequest('Bearer valid-api-key-12345'); - const res = createMockResponse(); + describe('Admin JWT Authentication', () => { + it('should call next() and populate req.user for valid Admin JWT', () => { + const token = generateAdminJwt({ id: 'admin-1', username: 'admin' }); + const req = createMockRequest(`Bearer ${token}`) as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); expect(next).toHaveBeenCalled(); expect(res.status).not.toHaveBeenCalled(); + expect(req.user?.username).toBe('admin'); }); }); - describe('Missing authentication', () => { - it('should return 401 when authorization header is missing', () => { - const req = createMockRequest(); - const res = createMockResponse(); + describe('Dynamic API Key Authentication', () => { + it('should call next() and populate req.apiKey for valid dynamic key', async () => { + const rawKey = 'sns_live_abcdef1234567890'; + const hashed = crypto.createHash('sha256').update(rawKey).digest('hex'); + + const mockKeyDoc = { + key_id: 'key-123', + name: 'Test Service', + key_hash: hashed, + status: 'active', + expires_at: null, + toObject: () => ({ + key_id: 'key-123', + name: 'Test Service', + status: 'active', + }), + }; + + mockFindOne.mockResolvedValueOnce(mockKeyDoc); + + const req = createMockRequest(`Bearer ${rawKey}`) as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + await authMiddleware.auth_middleware(req, res, next as NextFunction); + + expect(next).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect(req.apiKey?.name).toBe('Test Service'); + }); + + it('should return 401 when dynamic API key is not found or revoked', async () => { + mockFindOne.mockResolvedValueOnce(null); + + const req = createMockRequest('Bearer sns_live_unknownkey') as Request; + const res = createMockResponse() as Response; + const next = vi.fn(); + + await authMiddleware.auth_middleware(req, res, next as NextFunction); expect(res.status).toHaveBeenCalledWith(401); - expect(res.json).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('API KEY missing'), - }) - ); + expect(res.json).toHaveBeenCalledWith({ message: 'Invalid API KEY' }); expect(next).not.toHaveBeenCalled(); }); - it('should return 401 when Bearer prefix is missing', () => { - const req = createMockRequest('valid-api-key-12345'); - const res = createMockResponse(); + it('should return 401 when dynamic API key has expired', async () => { + const rawKey = 'sns_live_expiredkey'; + const mockKeyDoc = { + key_id: 'key-expired', + status: 'active', + expires_at: new Date(Date.now() - 10000), // past + toObject: () => ({ key_id: 'key-expired' }), + }; + + mockFindOne.mockResolvedValueOnce(mockKeyDoc); + + const req = createMockRequest(`Bearer ${rawKey}`) as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + await authMiddleware.auth_middleware(req, res, next as NextFunction); expect(res.status).toHaveBeenCalledWith(401); - expect(res.json).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining("'Bearer' is missing"), - }) - ); + expect(res.json).toHaveBeenCalledWith({ message: 'API key has expired' }); + expect(next).not.toHaveBeenCalled(); + }); + + it('should return 403 when dynamic API key accesses admin credential routes', async () => { + const rawKey = 'sns_live_testkey'; + const mockKeyDoc = { + key_id: 'key-123', + status: 'active', + expires_at: null, + toObject: () => ({ key_id: 'key-123' }), + }; + + mockFindOne.mockResolvedValueOnce(mockKeyDoc); + + const req = createMockRequest(`Bearer ${rawKey}`, '/api/admin/auth/setup') as Request; + const res = createMockResponse() as Response; + const next = vi.fn(); + + await authMiddleware.auth_middleware(req, res, next as NextFunction); + + expect(res.status).toHaveBeenCalledWith(403); + expect(next).not.toHaveBeenCalled(); }); }); - describe('Invalid authentication', () => { - it('should return 401 for invalid API key', () => { - const req = createMockRequest('Bearer wrong-api-key'); - const res = createMockResponse(); + describe('Missing or Invalid Authentication', () => { + it('should return 401 when authorization header is missing', () => { + const req = createMockRequest() as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); expect(res.status).toHaveBeenCalledWith(401); expect(res.json).toHaveBeenCalledWith( expect.objectContaining({ - message: 'Invalid API KEY', + message: expect.stringContaining('API KEY missing'), }) ); + expect(next).not.toHaveBeenCalled(); }); - it('should return 401 for API key with different length', () => { - const req = createMockRequest('Bearer short'); - const res = createMockResponse(); + it('should return 401 when Bearer prefix is missing', () => { + const req = createMockRequest('raw-string-without-bearer') as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); expect(res.status).toHaveBeenCalledWith(401); expect(res.json).toHaveBeenCalledWith( expect.objectContaining({ - message: 'Invalid API KEY', + message: expect.stringContaining("'Bearer' is missing"), }) ); + expect(next).not.toHaveBeenCalled(); }); - it('should return 401 for empty API key after Bearer', () => { - const req = createMockRequest('Bearer '); - const res = createMockResponse(); + it('should return 401 for unknown non-JWT non-dynamic token', () => { + const req = createMockRequest('Bearer random-legacy-token') as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); expect(res.status).toHaveBeenCalledWith(401); + expect(res.json).toHaveBeenCalledWith({ message: 'Invalid API KEY' }); + expect(next).not.toHaveBeenCalled(); }); }); - describe('Server configuration errors', () => { - it('should return 500 when NS_API_KEY is not configured', async () => { - mockApiKey = undefined; - vi.resetModules(); - const freshMiddleware = await import('../../../../src/api/middlewares/auth_middleware.js'); - - const req = createMockRequest('Bearer some-key'); - const res = createMockResponse(); + describe('Cookie Authentication', () => { + it('should authenticate via simplens_session cookie containing valid Admin JWT', () => { + const token = generateAdminJwt({ id: 'admin-1', username: 'admin' }); + const req = createMockRequest(undefined, '/api/notifications', `simplens_session=${token}`) as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - freshMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); - expect(res.status).toHaveBeenCalledWith(500); - expect(res.json).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'Server configuration error', - }) - ); + expect(next).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect(req.user?.username).toBe('admin'); }); - it('should return 500 when NS_API_KEY is empty string', async () => { - mockApiKey = ''; - vi.resetModules(); - const freshMiddleware = await import('../../../../src/api/middlewares/auth_middleware.js'); + it('should authenticate via simplens_session cookie among other cookies', () => { + const token = generateAdminJwt({ id: 'admin-1', username: 'admin' }); + const req = createMockRequest( + undefined, + '/api/notifications', + `theme=dark; simplens_session=${token}; lang=en` + ) as Request; + const res = createMockResponse() as Response; + const next = vi.fn(); + + authMiddleware.auth_middleware(req, res, next as NextFunction); + + expect(next).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect(req.user?.username).toBe('admin'); + }); - const req = createMockRequest('Bearer some-key'); - const res = createMockResponse(); + it('should return 401 when simplens_session cookie contains an invalid token', () => { + const req = createMockRequest( + undefined, + '/api/notifications', + 'simplens_session=invalid-cookie-token' + ) as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - freshMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); - expect(res.status).toHaveBeenCalledWith(500); + expect(res.status).toHaveBeenCalledWith(401); + expect(res.json).toHaveBeenCalledWith({ message: 'Invalid API KEY' }); + expect(next).not.toHaveBeenCalled(); }); }); }); diff --git a/tests/integration/api/notification.controller.test.ts b/tests/integration/api/notification.controller.test.ts index 97c08f5..42ab80c 100644 --- a/tests/integration/api/notification.controller.test.ts +++ b/tests/integration/api/notification.controller.test.ts @@ -129,6 +129,7 @@ vi.mock("../../../src/api/utils/utils.js", () => { InvalidProviderChannelError, InvalidContentSchemaError, ProviderNotFoundError, + updateApiKeyNotificationUsage: vi.fn(), }; }); diff --git a/tests/integration/api/settings.test.ts b/tests/integration/api/settings.test.ts index ba5808d..0f25fb1 100644 --- a/tests/integration/api/settings.test.ts +++ b/tests/integration/api/settings.test.ts @@ -4,6 +4,7 @@ import express from 'express'; import settings_router from '@src/api/routes/settings.routes.js'; import { auth_middleware } from '@src/api/middlewares/auth_middleware.js'; import { dynamicConfig } from '@src/config/dynamic-config.service.js'; +import { generateAdminJwt } from '@src/api/utils/jwt.utils.js'; vi.mock('@src/config/redis.config.js', () => ({ getRedisClient: vi.fn(() => ({ @@ -29,6 +30,7 @@ vi.mock('@src/database/models/system-config.models.js', () => ({ describe('Settings API Integration', () => { let app: express.Application; + const adminToken = generateAdminJwt({ id: 'admin-1', username: 'admin' }); beforeEach(() => { vi.clearAllMocks(); @@ -45,7 +47,7 @@ describe('Settings API Integration', () => { it('should return operational settings with valid authorization', async () => { const res = await request(app) .get('/api/settings') - .set('Authorization', `Bearer ${process.env.NS_API_KEY}`); + .set('Authorization', `Bearer ${adminToken}`); expect(res.status).toBe(200); expect(res.body.success).toBe(true); @@ -55,7 +57,7 @@ describe('Settings API Integration', () => { it('should update operational settings with valid authorization', async () => { const res = await request(app) .put('/api/settings') - .set('Authorization', `Bearer ${process.env.NS_API_KEY}`) + .set('Authorization', `Bearer ${adminToken}`) .send({ worker: { outbox_batch_size: 150, @@ -74,7 +76,7 @@ describe('Settings API Integration', () => { it('should reject update with invalid schema bounds', async () => { const res = await request(app) .put('/api/settings') - .set('Authorization', `Bearer ${process.env.NS_API_KEY}`) + .set('Authorization', `Bearer ${adminToken}`) .send({ worker: { outbox_batch_size: 99999, // exceeds max 2000 @@ -88,7 +90,7 @@ describe('Settings API Integration', () => { it('should reset operational settings to system defaults', async () => { const res = await request(app) .post('/api/settings/reset') - .set('Authorization', `Bearer ${process.env.NS_API_KEY}`); + .set('Authorization', `Bearer ${adminToken}`); expect(res.status).toBe(200); expect(res.body.success).toBe(true); diff --git a/tests/integration/mcp_server.test.ts b/tests/integration/mcp_server.test.ts index 5da2c43..de9b14c 100644 --- a/tests/integration/mcp_server.test.ts +++ b/tests/integration/mcp_server.test.ts @@ -103,9 +103,9 @@ describe('MCP Server Integration Tests', () => { await server.close(); }); - it('should register exactly 29 tools', async () => { + it('should register all expected tools', async () => { const toolsResult = await client.listTools(); - expect(toolsResult.tools.length).toBe(29); + expect(toolsResult.tools.length).toBe(54); const toolNames = toolsResult.tools.map(t => t.name); expect(toolNames).toContain('send_notification'); diff --git a/tests/unit/api/jwt.utils.test.ts b/tests/unit/api/jwt.utils.test.ts new file mode 100644 index 0000000..f66c7e9 --- /dev/null +++ b/tests/unit/api/jwt.utils.test.ts @@ -0,0 +1,43 @@ +/** + * Unit tests for jwt.utils.ts + */ + +import { describe, it, expect } from 'vitest'; +import { generateAdminJwt, verifyAdminJwt } from '@src/api/utils/jwt.utils.js'; + +describe('Admin JWT Utilities', () => { + it('should generate a valid JWT token for admin user', () => { + const admin = { id: 'admin-1', username: 'admin' }; + const token = generateAdminJwt(admin); + + expect(token).toBeDefined(); + expect(typeof token).toBe('string'); + expect(token.split('.')).toHaveLength(3); + }); + + it('should verify and decode a valid JWT token', () => { + const admin = { id: 'admin-42', username: 'superadmin' }; + const token = generateAdminJwt(admin); + + const decoded = verifyAdminJwt(token); + expect(decoded).not.toBeNull(); + expect(decoded?.sub).toBe('admin-42'); + expect(decoded?.username).toBe('superadmin'); + expect(decoded?.role).toBe('admin'); + }); + + it('should return null when token signature is tampered', () => { + const admin = { id: 'admin-1', username: 'admin' }; + const token = generateAdminJwt(admin); + const tamperedToken = token.slice(0, -5) + 'abcde'; + + const decoded = verifyAdminJwt(tamperedToken); + expect(decoded).toBeNull(); + }); + + it('should return null for malformed tokens', () => { + expect(verifyAdminJwt('not.a.valid.jwt')).toBeNull(); + expect(verifyAdminJwt('')).toBeNull(); + expect(verifyAdminJwt('random-string')).toBeNull(); + }); +}); diff --git a/tests/unit/api/utils.test.ts b/tests/unit/api/utils.test.ts index e1e3201..38a5b01 100644 --- a/tests/unit/api/utils.test.ts +++ b/tests/unit/api/utils.test.ts @@ -38,6 +38,12 @@ vi.mock("../../../src/database/models/notification-template.models.js", () => ({ }, })); +vi.mock("../../../src/database/models/api-key.models.js", () => ({ + default: { + updateOne: vi.fn().mockReturnValue({ catch: vi.fn() }), + }, +})); + vi.mock("../../../src/workers/utils/logger.js", () => ({ apiLogger: { info: vi.fn(), @@ -1283,4 +1289,62 @@ describe("API Utility Functions", () => { expect(outbox_model.insertMany).not.toHaveBeenCalled(); }); }); + + describe("updateApiKeyNotificationUsage", () => { + it("should do nothing if apiKey is undefined", async () => { + const api_key_model = ( + await import("../../../src/database/models/api-key.models.js") + ).default; + const { updateApiKeyNotificationUsage } = + await import("../../../src/api/utils/utils.js"); + + updateApiKeyNotificationUsage(undefined, [], 5); + + expect(api_key_model.updateOne).not.toHaveBeenCalled(); + }); + + it("should do nothing if createdCount is 0 or negative", async () => { + const api_key_model = ( + await import("../../../src/database/models/api-key.models.js") + ).default; + const { updateApiKeyNotificationUsage } = + await import("../../../src/api/utils/utils.js"); + + const mockApiKey = { key_id: "ak_test123" } as any; + + updateApiKeyNotificationUsage(mockApiKey, [], 0); + updateApiKeyNotificationUsage(mockApiKey, [], -1); + + expect(api_key_model.updateOne).not.toHaveBeenCalled(); + }); + + it("should update key usage with total notifications and channel breakdown", async () => { + const api_key_model = ( + await import("../../../src/database/models/api-key.models.js") + ).default; + const { updateApiKeyNotificationUsage } = + await import("../../../src/api/utils/utils.js"); + + const mockApiKey = { key_id: "ak_test123" } as any; + const mockNotifications = [ + { channel: "email" }, + { channel: "email" }, + { channel: "sms" }, + ] as any[]; + + updateApiKeyNotificationUsage(mockApiKey, mockNotifications, 3); + + expect(api_key_model.updateOne).toHaveBeenCalledWith( + { key_id: "ak_test123" }, + expect.objectContaining({ + $set: expect.objectContaining({ "usage.last_used_at": expect.any(Date) }), + $inc: { + "usage.total_notifications": 3, + "usage.by_channel.email": 2, + "usage.by_channel.sms": 1, + }, + }), + ); + }); + }); }); From bd67ff4e461ca79effcf1479034879302984b661 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Fri, 25 Sep 2026 15:12:01 +0530 Subject: [PATCH 30/46] feat: update credential management to replace AUTH_SECRET with JWT_SECRET and adjust related logic --- packages/onboard/README.md | 6 +++--- packages/onboard/src/__tests__/env-config.test.ts | 7 +++++-- packages/onboard/src/__tests__/validators.test.ts | 1 + packages/onboard/src/config/constants.ts | 3 +-- packages/onboard/src/env-config.ts | 10 ++++------ packages/onboard/src/index.ts | 11 ++++++----- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/packages/onboard/README.md b/packages/onboard/README.md index d14428c..bde3bed 100644 --- a/packages/onboard/README.md +++ b/packages/onboard/README.md @@ -85,15 +85,15 @@ npx @simplens/onboard --full --infra mongo kafka redis nginx --env default --bas This mode: - Requires `--full` flag to enable non-interactive mode - Requires `--env ` to specify environment mode -- **Auto-generates secure credentials** for `NS_API_KEY` and `AUTH_SECRET` (and `ADMIN_PASSWORD` for legacy SimpleNS <= 1.3.0) +- **Auto-generates secure credentials** for `JWT_SECRET` (and `AUTH_SECRET`/`NS_API_KEY`/`ADMIN_PASSWORD` for legacy SimpleNS <= 1.3.0) - **Auto-generates placeholder credentials** for plugins if `--plugin` is provided - **Auto-generates version value** for `VERSION` (default: `latest`) - All other options are optional with sensible defaults - Services are not auto-started (use `docker compose up -d` manually; fallback: `docker-compose up -d`) **⚠️ IMPORTANT**: Auto-generated credentials are **NOT secure for production**. After setup completes, you **must** update the following in your `.env` file: -- `NS_API_KEY` - API authentication key -- `AUTH_SECRET` - Session secret for dashboard +- `JWT_SECRET` - Secret key for API authentication tokens (for legacy <= 1.3.0, `AUTH_SECRET` was used for dashboard session) +- `NS_API_KEY` - API authentication key (legacy SimpleNS <= 1.3.0 only; for > 1.3.0, external API keys are created and managed dynamically in the Dashboard at `/keys`) - `ADMIN_PASSWORD` - Dashboard admin password (legacy SimpleNS <= 1.3.0 only; for > 1.3.0, the admin account is set up interactively via the Dashboard at `/setup` upon first boot) - Plugin credentials (if any plugins were installed) diff --git a/packages/onboard/src/__tests__/env-config.test.ts b/packages/onboard/src/__tests__/env-config.test.ts index 6c6bf07..de376a2 100644 --- a/packages/onboard/src/__tests__/env-config.test.ts +++ b/packages/onboard/src/__tests__/env-config.test.ts @@ -56,11 +56,12 @@ describe('loadEnvExample templates', () => { const envVars = await loadEnvExample(false); const keys = envVars.map(v => v.key); - expect(keys).toContain('NS_API_KEY'); + expect(keys).not.toContain('NS_API_KEY'); expect(keys).toContain('MONGO_URI'); expect(keys).toContain('BROKERS'); expect(keys).toContain('REDIS_URL'); - expect(keys).toContain('AUTH_SECRET'); + expect(keys).not.toContain('AUTH_SECRET'); + expect(keys).toContain('JWT_SECRET'); expect(keys).toContain('VERSION'); expect(keys).not.toContain('CORE_VERSION'); expect(keys).not.toContain('DASHBOARD_VERSION'); @@ -72,12 +73,14 @@ describe('loadEnvExample templates', () => { expect(keys).not.toContain('OUTBOX_BATCH_SIZE'); expect(keys).not.toContain('RECOVERY_BATCH_SIZE'); expect(keys).not.toContain('IDEMPOTENCY_TTL_SECONDS'); + expect(keys).not.toContain('HTTPS_COOKIE'); }); it('returns legacy variables including admin credentials and outbox settings for legacy (<= 1.3.0)', async () => { const envVars = await loadEnvExample(true); const keys = envVars.map(v => v.key); + expect(keys).toContain('NS_API_KEY'); expect(keys).toContain('VERSION'); expect(keys).not.toContain('CORE_VERSION'); expect(keys).not.toContain('DASHBOARD_VERSION'); diff --git a/packages/onboard/src/__tests__/validators.test.ts b/packages/onboard/src/__tests__/validators.test.ts index ec8a603..cd32cb5 100644 --- a/packages/onboard/src/__tests__/validators.test.ts +++ b/packages/onboard/src/__tests__/validators.test.ts @@ -177,6 +177,7 @@ describe('validators', () => { it('should reject short secrets', () => { expect(validateEnvValue('AUTH_SECRET', 'abc')).toBe(false); + expect(validateEnvValue('JWT_SECRET', 'abc')).toBe(false); }); it('should accept API keys with 8+ characters', () => { diff --git a/packages/onboard/src/config/constants.ts b/packages/onboard/src/config/constants.ts index 59357fa..22e364e 100644 --- a/packages/onboard/src/config/constants.ts +++ b/packages/onboard/src/config/constants.ts @@ -51,11 +51,10 @@ export const LEGACY_CRITICAL_ENV_KEYS = [ * Critical environment variables that always need user input (SimpleNS > 1.3.0) */ export const CRITICAL_ENV_KEYS = [ - 'NS_API_KEY', 'MONGO_URI', 'BROKERS', 'REDIS_URL', - 'AUTH_SECRET', + 'JWT_SECRET', 'VERSION', ]; diff --git a/packages/onboard/src/env-config.ts b/packages/onboard/src/env-config.ts index afb7eca..af64a75 100644 --- a/packages/onboard/src/env-config.ts +++ b/packages/onboard/src/env-config.ts @@ -32,7 +32,7 @@ export function isVersionGreaterThan130(version: string | undefined): boolean { if (minor < 3) return false; return patch > 0; } -const AUTO_GENERATE_ON_EMPTY_KEYS = ['NS_API_KEY', 'AUTH_SECRET', 'VERSION'] as const; +const AUTO_GENERATE_ON_EMPTY_KEYS = ['NS_API_KEY', 'AUTH_SECRET', 'JWT_SECRET', 'VERSION'] as const; function shouldAutoGenerateOnEmpty(key: string, fullMode: boolean): boolean { return !fullMode && AUTO_GENERATE_ON_EMPTY_KEYS.includes(key as (typeof AUTO_GENERATE_ON_EMPTY_KEYS)[number]); @@ -60,7 +60,7 @@ export function generateDefaultValue(key: string): string { if (key === 'NS_API_KEY') { return `ns_${generateSecureRandom(48)}`; } - if (key === 'AUTH_SECRET') { + if (key === 'AUTH_SECRET' || key === 'JWT_SECRET') { return generateSecureRandom(64); } if (key === 'ADMIN_PASSWORD') { @@ -131,8 +131,8 @@ NODE_ENV=production # ============================================ # API SERVER # ============================================ -NS_API_KEY= PORT=3000 +JWT_SECRET= # ============================================ # DATABASE @@ -165,13 +165,11 @@ VERSION=latest # ADMIN DASHBOARD # ============================================ AUTH_TRUST_HOST=true -AUTH_SECRET= API_BASE_URL=http://api:3000 WEBHOOK_HOST=dashboard WEBHOOK_PORT=3002 DASHBOARD_PORT=3002 BASE_PATH= -HTTPS_COOKIE=false `; export const LEGACY_ENV_TEMPLATE = ` @@ -532,7 +530,7 @@ export async function promptEnvVariables( * Get suggested value for specific keys */ function getSuggestedValue(key: string): string { - if (key === 'NS_API_KEY' || key === 'AUTH_SECRET') { + if (key === 'NS_API_KEY' || key === 'AUTH_SECRET' || key === 'JWT_SECRET') { return `Replace with: openssl rand -base64 32 (or) Press to auto-generate value`; } if (key === 'NODE_ENV') { diff --git a/packages/onboard/src/index.ts b/packages/onboard/src/index.ts index e028f54..9faf05e 100644 --- a/packages/onboard/src/index.ts +++ b/packages/onboard/src/index.ts @@ -543,7 +543,7 @@ async function main() { } else { logWarning( '⚠️ Auto-generated credentials in .env file. ' + - 'Please update NS_API_KEY and AUTH_SECRET before deploying to production!' + 'Please update JWT_SECRET before deploying to production!' ); } } @@ -736,12 +736,13 @@ async function main() { // In full mode, show a comprehensive security warning if (options.full) { - const credentialWarnings = [ - ' • NS_API_KEY - API authentication key', - ' • AUTH_SECRET - Session secret for dashboard', - ]; + const credentialWarnings: string[] = []; if (isLegacy) { + credentialWarnings.push(' • NS_API_KEY - API authentication key'); + credentialWarnings.push(' • AUTH_SECRET - Session secret for dashboard'); credentialWarnings.push(' • ADMIN_PASSWORD - Dashboard admin password'); + } else { + credentialWarnings.push(' • JWT_SECRET - Secret key for API authentication tokens'); } if (pluginCredentialKeys.length > 0) { From f59e5b0ef08191ce41a123c28d2782308fc75bb6 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Fri, 25 Sep 2026 17:02:26 +0530 Subject: [PATCH 31/46] feat: add health check endpoints and implement health probe server - Created health check routes: /api/health, /api/healthz (liveness), and /api/readyz (readiness). - Implemented health check logic in health.controller.js to assess MongoDB and Redis connectivity. - Integrated health probe server for Kubernetes with readiness and liveness checks. - Enhanced delayed and status consumers with health check capabilities, including crash detection and heartbeat monitoring. - Added unit tests for health endpoints and consumer health checks to ensure reliability. --- src/api/controllers/health.controller.ts | 68 +++++++ src/api/routes/health.routes.ts | 15 ++ src/api/server.ts | 61 ++++-- src/processors/delayed/delayed.consumer.ts | 51 +++++ src/processors/delayed/delayed.processor.ts | 54 ++++-- src/processors/delayed/dlq.status.ts | 7 + src/processors/delayed/target.producer.ts | 7 + src/processors/shared/delayed.producer.ts | 7 + src/processors/shared/status.producer.ts | 7 + src/processors/unified/unified.processor.ts | 14 +- src/types/schemas.ts | 7 + src/types/types.ts | 12 ++ src/utils/k8s-health-probe.ts | 44 +++-- src/workers/consumers/status.consumer.ts | 51 +++++ src/workers/producers/background.producer.ts | 7 + src/workers/recovery/recovery.service.ts | 10 +- src/workers/worker.ts | 38 +++- tests/unit/api/health.test.ts | 180 ++++++++++++++++++ .../delayed-consumer-health.test.ts | 92 +++++++++ tests/unit/utils/k8s-health-probe.test.ts | 155 +++++++++++++++ .../workers/status-consumer-health.test.ts | 94 +++++++++ 21 files changed, 919 insertions(+), 62 deletions(-) create mode 100644 src/api/controllers/health.controller.ts create mode 100644 src/api/routes/health.routes.ts create mode 100644 tests/unit/api/health.test.ts create mode 100644 tests/unit/processors/delayed-consumer-health.test.ts create mode 100644 tests/unit/utils/k8s-health-probe.test.ts create mode 100644 tests/unit/workers/status-consumer-health.test.ts diff --git a/src/api/controllers/health.controller.ts b/src/api/controllers/health.controller.ts new file mode 100644 index 0000000..bd37974 --- /dev/null +++ b/src/api/controllers/health.controller.ts @@ -0,0 +1,68 @@ +import type { Request, Response } from 'express'; +import mongoose from 'mongoose'; +import { getRedisClient } from '@src/config/redis.config.js'; +import { runHealthChecks } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeCheck, ApiHealthResponse } from '@src/types/types.js'; + +let isAppShuttingDown = false; + +export const setAppShuttingDown = (value: boolean): void => { + isAppShuttingDown = value; +}; + +export const getApiReadinessChecks = (): HealthProbeCheck[] => [ + { + name: 'mongodb', + check: () => !isAppShuttingDown && mongoose.connection.readyState === 1 + }, + { + name: 'redis', + check: () => { + if (isAppShuttingDown) return false; + try { + return getRedisClient().status === 'ready'; + } catch { + return false; + } + } + } +]; + +export const getApiLivenessChecks = (): HealthProbeCheck[] => [ + { + name: 'process', + check: () => !isAppShuttingDown + } +]; + +export const getHealth = async (req: Request, res: Response): Promise => { + const { ok, results } = await runHealthChecks(getApiReadinessChecks()); + const payload: ApiHealthResponse = { + status: ok ? 'healthy' : 'unhealthy', + timestamp: new Date().toISOString(), + checks: results + }; + res.status(ok ? 200 : 503).json(payload); +}; + +export const getLiveness = async (req: Request, res: Response): Promise => { + const { ok, results } = await runHealthChecks(getApiLivenessChecks()); + const payload: ApiHealthResponse = { + service: 'api', + status: ok ? 'healthy' : 'unhealthy', + timestamp: new Date().toISOString(), + checks: results + }; + res.status(ok ? 200 : 503).json(payload); +}; + +export const getReadiness = async (req: Request, res: Response): Promise => { + const { ok, results } = await runHealthChecks(getApiReadinessChecks()); + const payload: ApiHealthResponse = { + service: 'api', + status: ok ? 'ready' : 'not_ready', + timestamp: new Date().toISOString(), + checks: results + }; + res.status(ok ? 200 : 503).json(payload); +}; diff --git a/src/api/routes/health.routes.ts b/src/api/routes/health.routes.ts new file mode 100644 index 0000000..bde14d3 --- /dev/null +++ b/src/api/routes/health.routes.ts @@ -0,0 +1,15 @@ +import { Router } from 'express'; +import { getHealth, getLiveness, getReadiness } from '../controllers/health.controller.js'; + +const router = Router(); + +// GET /api/health +router.get('/health', getHealth); + +// GET /api/healthz (liveness) +router.get('/healthz', getLiveness); + +// GET /api/readyz (readiness) +router.get('/readyz', getReadiness); + +export default router; diff --git a/src/api/server.ts b/src/api/server.ts index 0159dd5..a37276b 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -16,7 +16,16 @@ import dashboard_router from './routes/dashboard.routes.js'; import settings_router from './routes/settings.routes.js'; import admin_auth_router from './routes/admin-auth.routes.js'; import api_keys_router from './routes/api-key.routes.js'; +import health_router from './routes/health.routes.js'; +import { + getApiReadinessChecks, + getApiLivenessChecks, + setAppShuttingDown +} from './controllers/health.controller.js'; import { dynamicConfig } from '@src/config/dynamic-config.service.js'; +import { disconnectRedis } from '@src/config/redis.config.js'; +import { createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeServer } from '@src/types/types.js'; import { auth_middleware } from './middlewares/auth_middleware.js'; import http from 'http'; import helmet from 'helmet'; @@ -39,6 +48,8 @@ import "@src/admin-alerts/channels/discord.channel.js"; import "@src/admin-alerts/channels/telegram.channel.js"; const app = express(); +let isShuttingDown = false; +let probeServer: HealthProbeServer | null = null; //implement rate limiter with REDIS later @@ -58,14 +69,8 @@ app.get("/api", (req: Request, res: Response) => { return; }); -// Health check endpoint for Docker/Kubernetes -app.get("/api/health", (req: Request, res: Response) => { - res.status(200).json({ - status: "healthy", - timestamp: new Date().toISOString() - }); - return; -}); +// Health check endpoints for Docker/Kubernetes and API consumers +app.use('/api', health_router); app.use('/api/notification', auth_middleware, notification_router); app.use('/api/notifications', auth_middleware, notifications_management_router); @@ -110,20 +115,44 @@ const start_server = async () => { const server = http.createServer(app); server.listen(env.PORT, () => logger.success(`Notification Service running at http://localhost:${env.PORT}`)); + + // 5. Start health probe server for Kubernetes + probeServer = createHealthProbeServer({ + serviceName: 'api', + readinessChecks: getApiReadinessChecks(), + livenessChecks: getApiLivenessChecks() + }); + await probeServer.start(); + const gracefulShutdown = async (err?: Error, reason?: string) => { - logger.error('Shutting down server', { reason: reason ?? '', error: err?.message ?? '' }); + if (isShuttingDown) return; + isShuttingDown = true; + setAppShuttingDown(true); + logger.info(`Shutting down server (reason: ${reason ?? 'signal'})`); try { - server?.close(() => { - logger.info('HTTP server closed'); - }); + if (probeServer) { + await probeServer.stop(); + } + if (server) { + await new Promise((resolve) => { + server.close(() => { + logger.info('HTTP server closed'); + resolve(); + }); + }); + } + await disconnectRedis(); await db.disconnect(); } catch (e) { logger.error('Error during graceful shutdown', e); } finally { - process.exit(1); + process.exit(err ? 1 : 0); } }; + process.on('SIGTERM', () => void gracefulShutdown(undefined, 'SIGTERM')); + process.on('SIGINT', () => void gracefulShutdown(undefined, 'SIGINT')); + process.on('uncaughtException', (err) => { logger.error('Uncaught exception:', err); @@ -134,7 +163,7 @@ const start_server = async () => { `Action: Review error handling. Check for async operations without try-catch.`, { severity: 'critical' }); - gracefulShutdown(err, 'uncaughtException'); + void gracefulShutdown(err, 'uncaughtException'); }); process.on('unhandledRejection', (reason) => { @@ -146,7 +175,7 @@ const start_server = async () => { `Action: Add .catch() handlers to promises. Review async/await error handling.`, { severity: 'critical' }); - gracefulShutdown(undefined, 'unhandledRejection'); + void gracefulShutdown(undefined, 'unhandledRejection'); }); server.on('error', (err) => { @@ -157,7 +186,7 @@ const start_server = async () => { `Port: ${env.PORT}\n` + `Action: Check if port is in use. Review server logs for stack trace.`, { severity: 'critical' }); - gracefulShutdown(err, 'serverError'); + void gracefulShutdown(err, 'serverError'); }); } catch (err) { diff --git a/src/processors/delayed/delayed.consumer.ts b/src/processors/delayed/delayed.consumer.ts index 47c6a6e..8cdd714 100644 --- a/src/processors/delayed/delayed.consumer.ts +++ b/src/processors/delayed/delayed.consumer.ts @@ -17,6 +17,10 @@ const state: KafkaConsumerState = { isConsuming: false }; +let lastHeartbeat = Date.now(); +let hasCrashed = false; +let crashReason: string | undefined = undefined; + /** * Process a delayed notification message * Returns true if message was successfully added to queue @@ -71,6 +75,40 @@ export const startDelayedConsumer = async (): Promise => { await state.consumer.connect(); logger.success('Delayed consumer connected'); + lastHeartbeat = Date.now(); + hasCrashed = false; + crashReason = undefined; + + if (typeof state.consumer.on === 'function' && state.consumer.events) { + state.consumer.on(state.consumer.events.HEARTBEAT, () => { + lastHeartbeat = Date.now(); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + state.consumer.on(state.consumer.events.CRASH, (event: any) => { + hasCrashed = true; + state.isConsuming = false; + const crashError = event?.payload?.error; + crashReason = crashError ? String(crashError.message || crashError) : 'Unknown crash'; + logger.error(`Delayed consumer crashed: ${crashReason}`); + }); + + state.consumer.on(state.consumer.events.STOP, () => { + state.isConsuming = false; + logger.warn('Delayed consumer stopped event received'); + }); + + state.consumer.on(state.consumer.events.DISCONNECT, () => { + state.isConsuming = false; + logger.warn('Delayed consumer disconnected'); + }); + + state.consumer.on(state.consumer.events.CONNECT, () => { + state.isConsuming = true; + lastHeartbeat = Date.now(); + }); + } + await state.consumer.subscribe({ topic: CORE_TOPICS.delayed_notification, fromBeginning: false @@ -82,6 +120,7 @@ export const startDelayedConsumer = async (): Promise => { await state.consumer.run({ autoCommit: false, eachMessage: async (payload) => { + lastHeartbeat = Date.now(); try { const success = await processDelayedMessage(payload); @@ -117,6 +156,7 @@ export const stopDelayedConsumer = async (): Promise => { } state.isConsuming = false; + hasCrashed = false; try { await state.consumer.stop(); @@ -135,3 +175,14 @@ export const stopDelayedConsumer = async (): Promise => { export const isConsumerActive = (): boolean => { return state.isConsuming && state.consumer !== null; }; + +/** + * Check if delayed consumer is healthy (running, not crashed, heartbeat fresh) + */ +export const isDelayedConsumerHealthy = (maxHeartbeatStalenessMs = 90000): boolean => { + if (!state.consumer || !state.isConsuming || hasCrashed) { + return false; + } + const isStale = (Date.now() - lastHeartbeat) > maxHeartbeatStalenessMs; + return !isStale; +}; diff --git a/src/processors/delayed/delayed.processor.ts b/src/processors/delayed/delayed.processor.ts index d6143a3..13f1490 100644 --- a/src/processors/delayed/delayed.processor.ts +++ b/src/processors/delayed/delayed.processor.ts @@ -10,12 +10,13 @@ * - Atomic Lua script prevents duplicate processing across multiple workers */ +import mongoose from 'mongoose'; import { connectMongoDB } from '@src/config/db.config.js'; import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { connectRedis, disconnectRedis } from '@src/config/redis.config.js'; -import { initTargetProducer, disconnectTargetProducer } from './target.producer.js'; -import { initDLQStatusProducer, disconnectDLQStatusProducer } from './dlq.status.js'; -import { startDelayedConsumer, stopDelayedConsumer, isConsumerActive } from './delayed.consumer.js'; +import { initTargetProducer, disconnectTargetProducer, isTargetProducerActive } from './target.producer.js'; +import { initDLQStatusProducer, disconnectDLQStatusProducer, isDLQProducerActive } from './dlq.status.js'; +import { startDelayedConsumer, stopDelayedConsumer, isDelayedConsumerHealthy } from './delayed.consumer.js'; import { startDelayedPoller, stopDelayedPoller, isPollerActive } from './delayed.poller.js'; import { delayedWorkerLogger as logger } from '@src/workers/utils/logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; @@ -156,18 +157,47 @@ const main = async (): Promise => { probeServer = createHealthProbeServer({ serviceName: 'delayed-processor', readinessChecks: [ - { name: 'redis', check: () => { - try { - return getRedisClient().status === 'ready'; - } catch { - return false; + { + name: 'mongodb', + check: () => !isShuttingDown && mongoose.connection.readyState === 1 + }, + { + name: 'redis', + check: () => { + if (isShuttingDown) return false; + try { + return getRedisClient().status === 'ready'; + } catch { + return false; + } } - }}, - { name: 'delayed_consumer', check: () => isConsumerActive() }, - { name: 'delayed_poller', check: () => isPollerActive() } + }, + { + name: 'delayed_consumer', + check: () => !isShuttingDown && isDelayedConsumerHealthy() + }, + { + name: 'delayed_poller', + check: () => !isShuttingDown && isPollerActive() + }, + { + name: 'kafka_producers', + check: () => !isShuttingDown && isTargetProducerActive() && isDLQProducerActive() + } ], livenessChecks: [ - { name: 'process', check: () => true } + { + name: 'process', + check: () => !isShuttingDown + }, + { + name: 'delayed_consumer', + check: () => !isShuttingDown && isDelayedConsumerHealthy() + }, + { + name: 'delayed_poller', + check: () => !isShuttingDown && isPollerActive() + } ] }); await probeServer.start(); diff --git a/src/processors/delayed/dlq.status.ts b/src/processors/delayed/dlq.status.ts index 42ec57b..01f8fe9 100644 --- a/src/processors/delayed/dlq.status.ts +++ b/src/processors/delayed/dlq.status.ts @@ -87,3 +87,10 @@ export const disconnectDLQStatusProducer = async (): Promise => { logger.info('DLQ status producer disconnected'); } }; + +/** + * Check if DLQ status producer is active + */ +export const isDLQProducerActive = (): boolean => { + return producer !== null; +}; diff --git a/src/processors/delayed/target.producer.ts b/src/processors/delayed/target.producer.ts index 51235fa..86fce9f 100644 --- a/src/processors/delayed/target.producer.ts +++ b/src/processors/delayed/target.producer.ts @@ -63,3 +63,10 @@ export const disconnectTargetProducer = async (): Promise => { logger.info('Target producer disconnected'); } }; + +/** + * Check if target producer is active + */ +export const isTargetProducerActive = (): boolean => { + return producer !== null; +}; diff --git a/src/processors/shared/delayed.producer.ts b/src/processors/shared/delayed.producer.ts index 533a05b..4ef2865 100644 --- a/src/processors/shared/delayed.producer.ts +++ b/src/processors/shared/delayed.producer.ts @@ -96,3 +96,10 @@ export const disconnectDelayedProducer = async (): Promise => { producer = null; } }; + +/** + * Check if delayed producer is active + */ +export const isDelayedProducerActive = (): boolean => { + return producer !== null; +}; diff --git a/src/processors/shared/status.producer.ts b/src/processors/shared/status.producer.ts index 2a1334c..47fe1b0 100644 --- a/src/processors/shared/status.producer.ts +++ b/src/processors/shared/status.producer.ts @@ -53,3 +53,10 @@ export const disconnectStatusProducer = async (): Promise => { producer = null; } }; + +/** + * Check if status producer is active + */ +export const isStatusProducerActive = (): boolean => { + return producer !== null; +}; diff --git a/src/processors/unified/unified.processor.ts b/src/processors/unified/unified.processor.ts index 37c8c30..56cccd6 100644 --- a/src/processors/unified/unified.processor.ts +++ b/src/processors/unified/unified.processor.ts @@ -7,11 +7,12 @@ * - Multi: PROCESSOR_CHANNEL=all (or omit to process all configured channels) */ +import mongoose from 'mongoose'; import { connectMongoDB } from '@src/config/db.config.js'; import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { connectRedis, disconnectRedis } from '@src/config/redis.config.js'; -import { initStatusProducer, disconnectStatusProducer } from '@src/processors/shared/status.producer.js'; -import { initDelayedProducer, disconnectDelayedProducer } from '@src/processors/shared/delayed.producer.js'; +import { initStatusProducer, disconnectStatusProducer, isStatusProducerActive } from '@src/processors/shared/status.producer.js'; +import { initDelayedProducer, disconnectDelayedProducer, isDelayedProducerActive } from '@src/processors/shared/delayed.producer.js'; import { PluginRegistry, PluginSyncService, @@ -283,7 +284,10 @@ const main = async (): Promise => { probeServer = createHealthProbeServer({ serviceName: 'unified-processor', readinessChecks: [ - { name: 'mongodb', check: () => !isShuttingDown && dbConnection !== null }, + { + name: 'mongodb', + check: () => !isShuttingDown && mongoose.connection.readyState === 1 + }, { name: 'redis', check: () => { @@ -295,6 +299,10 @@ const main = async (): Promise => { } } }, + { + name: 'kafka_producers', + check: () => !isShuttingDown && isStatusProducerActive() && isDelayedProducerActive() + }, { name: 'kafka_consumers', check: () => { diff --git a/src/types/schemas.ts b/src/types/schemas.ts index 49cda84..73d16f6 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -726,3 +726,10 @@ export const createApiKeyResponseSchema = z.object({ raw_key: z.string(), }); +export const apiHealthResponseSchema = z.object({ + service: z.string().optional(), + status: z.enum(['healthy', 'unhealthy', 'ready', 'not_ready']), + timestamp: z.string(), + checks: z.record(z.string(), z.boolean()), +}); + diff --git a/src/types/types.ts b/src/types/types.ts index 7cd473d..e6771ed 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -532,6 +532,18 @@ export interface HealthProbeServer { stop: () => Promise; } +export interface HealthCheckResultDetails { + ok: boolean; + results: Record; +} + +export interface ApiHealthResponse { + service?: string; + status: 'healthy' | 'unhealthy' | 'ready' | 'not_ready'; + timestamp: string; + checks: Record; +} + // ============================================================================ // PROCESSOR CONSUMER HEALTH TYPES // ============================================================================ diff --git a/src/utils/k8s-health-probe.ts b/src/utils/k8s-health-probe.ts index f289e83..9395d56 100644 --- a/src/utils/k8s-health-probe.ts +++ b/src/utils/k8s-health-probe.ts @@ -8,7 +8,27 @@ */ import http from 'http'; -import type { HealthProbeOptions, HealthProbeServer } from '@src/types/types.js'; +import type { HealthCheckResultDetails, HealthProbeCheck, HealthProbeOptions, HealthProbeServer } from '@src/types/types.js'; + +export const runHealthChecks = async ( + checks: HealthProbeCheck[] = [] +): Promise => { + const results: Record = {}; + let allOk = true; + + for (const item of checks) { + try { + const passed = await item.check(); + results[item.name] = Boolean(passed); + if (!passed) allOk = false; + } catch { + results[item.name] = false; + allOk = false; + } + } + + return { ok: allOk, results }; +}; export const createHealthProbeServer = (options: HealthProbeOptions): HealthProbeServer => { const port = options.port ?? parseInt(process.env.PROBE_PORT || '8080', 10); @@ -18,30 +38,12 @@ export const createHealthProbeServer = (options: HealthProbeOptions): HealthProb let server: http.Server | null = null; - const runChecks = async (checks: typeof readinessChecks): Promise<{ ok: boolean; results: Record }> => { - const results: Record = {}; - let allOk = true; - - for (const item of checks) { - try { - const passed = await item.check(); - results[item.name] = passed; - if (!passed) allOk = false; - } catch { - results[item.name] = false; - allOk = false; - } - } - - return { ok: allOk, results }; - }; - const requestListener: http.RequestListener = async (req, res) => { const url = req.url || ''; // Liveness probe if (url === '/healthz' || url === '/livez') { - const { ok, results } = await runChecks(livenessChecks); + const { ok, results } = await runHealthChecks(livenessChecks); const status = ok ? 200 : 503; res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ @@ -55,7 +57,7 @@ export const createHealthProbeServer = (options: HealthProbeOptions): HealthProb // Readiness probe if (url === '/readyz' || url === '/ready') { - const { ok, results } = await runChecks(readinessChecks); + const { ok, results } = await runHealthChecks(readinessChecks); const status = ok ? 200 : 503; res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ diff --git a/src/workers/consumers/status.consumer.ts b/src/workers/consumers/status.consumer.ts index f82842f..10a2a1d 100644 --- a/src/workers/consumers/status.consumer.ts +++ b/src/workers/consumers/status.consumer.ts @@ -16,6 +16,10 @@ const state: KafkaConsumerState = { isConsuming: false }; +let lastHeartbeat = Date.now(); +let hasCrashed = false; +let crashReason: string | undefined = undefined; + /** * Map status from external format to internal notification status */ @@ -172,6 +176,40 @@ export const startStatusConsumer = async (): Promise => { await state.consumer.connect(); logger.info("Status consumer connected"); + lastHeartbeat = Date.now(); + hasCrashed = false; + crashReason = undefined; + + if (typeof state.consumer.on === 'function' && state.consumer.events) { + state.consumer.on(state.consumer.events.HEARTBEAT, () => { + lastHeartbeat = Date.now(); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + state.consumer.on(state.consumer.events.CRASH, (event: any) => { + hasCrashed = true; + state.isConsuming = false; + const crashError = event?.payload?.error; + crashReason = crashError ? String(crashError.message || crashError) : 'Unknown crash'; + logger.error(`Status consumer crashed: ${crashReason}`); + }); + + state.consumer.on(state.consumer.events.STOP, () => { + state.isConsuming = false; + logger.warn('Status consumer stopped event received'); + }); + + state.consumer.on(state.consumer.events.DISCONNECT, () => { + state.isConsuming = false; + logger.warn('Status consumer disconnected'); + }); + + state.consumer.on(state.consumer.events.CONNECT, () => { + state.isConsuming = true; + lastHeartbeat = Date.now(); + }); + } + await state.consumer.subscribe({ topic: CORE_TOPICS.notification_status, fromBeginning: false @@ -183,6 +221,7 @@ export const startStatusConsumer = async (): Promise => { await state.consumer.run({ autoCommit: false, eachMessage: async (payload) => { + lastHeartbeat = Date.now(); try { const result = await processStatusMessage(payload); @@ -224,6 +263,7 @@ export const stopStatusConsumer = async (): Promise => { logger.info("Stopping status consumer..."); state.isConsuming = false; + hasCrashed = false; try { await state.consumer.stop(); @@ -242,3 +282,14 @@ export const stopStatusConsumer = async (): Promise => { export const isStatusConsumerRunning = (): boolean => { return state.consumer !== null && state.isConsuming; }; + +/** + * Check if the status consumer is healthy (running, not crashed, heartbeat fresh) + */ +export const isStatusConsumerHealthy = (maxHeartbeatStalenessMs = 90000): boolean => { + if (!state.consumer || !state.isConsuming || hasCrashed) { + return false; + } + const isStale = (Date.now() - lastHeartbeat) > maxHeartbeatStalenessMs; + return !isStale; +}; diff --git a/src/workers/producers/background.producer.ts b/src/workers/producers/background.producer.ts index 0ca1864..5d18f3d 100644 --- a/src/workers/producers/background.producer.ts +++ b/src/workers/producers/background.producer.ts @@ -218,3 +218,10 @@ export const sendStatusOutboxEvents = async (statusEntries: status_outbox[]): Pr return { successCount, failedCount }; }; + +/** + * Check if the Kafka producer is initialized and active + */ +export const isProducerActive = (): boolean => { + return producer !== null; +}; diff --git a/src/workers/recovery/recovery.service.ts b/src/workers/recovery/recovery.service.ts index 4397a33..f2b6502 100644 --- a/src/workers/recovery/recovery.service.ts +++ b/src/workers/recovery/recovery.service.ts @@ -16,7 +16,7 @@ import mongoose from 'mongoose'; import { env } from '@src/config/env.config.js'; import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { connectRedis, disconnectRedis, getRedisClient } from '@src/config/redis.config.js'; -import { startRecoveryCron, stopRecoveryCron, setHealthChecker } from './recovery.cron.js'; +import { startRecoveryCron, stopRecoveryCron, setHealthChecker, isRecoveryCronRunning } from './recovery.cron.js'; import { recoveryLogger as logger, flushLogs } from '@src/workers/utils/logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; import { createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; @@ -292,11 +292,13 @@ const main = async (): Promise => { probeServer = createHealthProbeServer({ serviceName: 'recovery-service', readinessChecks: [ - { name: 'mongodb', check: () => isMongoHealthy() }, - { name: 'redis', check: () => isRedisHealthy() } + { name: 'mongodb', check: () => !isShuttingDown && isMongoHealthy() }, + { name: 'redis', check: () => !isShuttingDown && isRedisHealthy() }, + { name: 'recovery_cron', check: () => !isShuttingDown && isRecoveryCronRunning() } ], livenessChecks: [ - { name: 'process', check: () => true } + { name: 'process', check: () => !isShuttingDown }, + { name: 'recovery_cron', check: () => !isShuttingDown && isRecoveryCronRunning() } ] }); await probeServer.start(); diff --git a/src/workers/worker.ts b/src/workers/worker.ts index d2f9247..58c0fd3 100644 --- a/src/workers/worker.ts +++ b/src/workers/worker.ts @@ -1,8 +1,9 @@ +import mongoose from "mongoose"; import { connectMongoDB } from "@src/config/db.config.js"; import { dynamicConfig } from "@src/config/dynamic-config.service.js"; -import { initProducer, disconnectProducer } from "@src/workers/producers/background.producer.js"; -import { startCronJobs, stopCronJobs } from "@src/workers/cron/background.cron.js"; -import { startStatusConsumer, stopStatusConsumer, isStatusConsumerRunning } from "@src/workers/consumers/status.consumer.js"; +import { initProducer, disconnectProducer, isProducerActive } from "@src/workers/producers/background.producer.js"; +import { startCronJobs, stopCronJobs, isCronRunning } from "@src/workers/cron/background.cron.js"; +import { startStatusConsumer, stopStatusConsumer, isStatusConsumerHealthy } from "@src/workers/consumers/status.consumer.js"; import { workerLogger as logger } from "@src/workers/utils/logger.js"; import { AdminAlertService } from "@src/admin-alerts/admin-alert.service.js"; import { createHealthProbeServer } from "@src/utils/k8s-health-probe.js"; @@ -129,11 +130,36 @@ const main = async (): Promise => { probeServer = createHealthProbeServer({ serviceName: 'worker', readinessChecks: [ - { name: 'mongodb', check: () => dbConnection !== null }, - { name: 'status_consumer', check: () => isStatusConsumerRunning() } + { + name: 'mongodb', + check: () => !isShuttingDown && mongoose.connection.readyState === 1 + }, + { + name: 'status_consumer', + check: () => !isShuttingDown && isStatusConsumerHealthy() + }, + { + name: 'cron_jobs', + check: () => !isShuttingDown && isCronRunning() + }, + { + name: 'kafka_producer', + check: () => !isShuttingDown && isProducerActive() + } ], livenessChecks: [ - { name: 'process', check: () => true } + { + name: 'process', + check: () => !isShuttingDown + }, + { + name: 'status_consumer', + check: () => !isShuttingDown && isStatusConsumerHealthy() + }, + { + name: 'cron_jobs', + check: () => !isShuttingDown && isCronRunning() + } ] }); await probeServer.start(); diff --git a/tests/unit/api/health.test.ts b/tests/unit/api/health.test.ts new file mode 100644 index 0000000..81ace5a --- /dev/null +++ b/tests/unit/api/health.test.ts @@ -0,0 +1,180 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import mongoose from 'mongoose'; +import health_router from '@src/api/routes/health.routes.js'; +import { + getLiveness, + getReadiness, + setAppShuttingDown +} from '@src/api/controllers/health.controller.js'; +import { getRedisClient } from '@src/config/redis.config.js'; + +vi.mock('@src/config/redis.config.js', () => ({ + getRedisClient: vi.fn(), +})); + +describe('API Health Endpoints', () => { + let app: express.Application; + + beforeEach(() => { + vi.clearAllMocks(); + setAppShuttingDown(false); + + app = express(); + app.use('/api', health_router); + }); + + afterEach(() => { + setAppShuttingDown(false); + }); + + describe('GET /api/health', () => { + it('should return 200 and healthy when MongoDB and Redis are connected', async () => { + // Mock mongoose connection readyState = 1 (connected) + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(1); + + // Mock Redis status = 'ready' + vi.mocked(getRedisClient).mockReturnValue({ + status: 'ready', + } as unknown as ReturnType); + + const res = await request(app).get('/api/health'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + status: 'healthy', + timestamp: expect.any(String), + checks: { + mongodb: true, + redis: true, + }, + }); + }); + + it('should return 503 and unhealthy when MongoDB is disconnected', async () => { + // Mock mongoose connection readyState = 0 (disconnected) + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(0); + + vi.mocked(getRedisClient).mockReturnValue({ + status: 'ready', + } as unknown as ReturnType); + + const res = await request(app).get('/api/health'); + + expect(res.status).toBe(503); + expect(res.body).toEqual({ + status: 'unhealthy', + timestamp: expect.any(String), + checks: { + mongodb: false, + redis: true, + }, + }); + }); + + it('should return 503 and unhealthy when Redis is disconnected', async () => { + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(1); + + vi.mocked(getRedisClient).mockReturnValue({ + status: 'close', + } as unknown as ReturnType); + + const res = await request(app).get('/api/health'); + + expect(res.status).toBe(503); + expect(res.body).toEqual({ + status: 'unhealthy', + timestamp: expect.any(String), + checks: { + mongodb: true, + redis: false, + }, + }); + }); + + it('should return 503 when the app is shutting down', async () => { + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(1); + vi.mocked(getRedisClient).mockReturnValue({ + status: 'ready', + } as unknown as ReturnType); + + setAppShuttingDown(true); + + const res = await request(app).get('/api/health'); + + expect(res.status).toBe(503); + expect(res.body.status).toBe('unhealthy'); + expect(res.body.checks.mongodb).toBe(false); + expect(res.body.checks.redis).toBe(false); + }); + }); + + describe('GET /api/healthz (liveness)', () => { + it('should return 200 when process is not shutting down', async () => { + const res = await request(app).get('/api/healthz'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + service: 'api', + status: 'healthy', + timestamp: expect.any(String), + checks: { + process: true, + }, + }); + }); + + it('should return 503 when process is shutting down', async () => { + setAppShuttingDown(true); + + const res = await request(app).get('/api/healthz'); + + expect(res.status).toBe(503); + expect(res.body).toEqual({ + service: 'api', + status: 'unhealthy', + timestamp: expect.any(String), + checks: { + process: false, + }, + }); + }); + }); + + describe('GET /api/readyz (readiness)', () => { + it('should return 200 and ready when all checks pass', async () => { + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(1); + vi.mocked(getRedisClient).mockReturnValue({ + status: 'ready', + } as unknown as ReturnType); + + const res = await request(app).get('/api/readyz'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + service: 'api', + status: 'ready', + timestamp: expect.any(String), + checks: { + mongodb: true, + redis: true, + }, + }); + }); + + it('should return 503 when readiness checks fail', async () => { + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(0); + vi.mocked(getRedisClient).mockReturnValue({ + status: 'ready', + } as unknown as ReturnType); + + const res = await request(app).get('/api/readyz'); + + expect(res.status).toBe(503); + expect(res.body.status).toBe('not_ready'); + expect(res.body.checks.mongodb).toBe(false); + expect(res.body.checks.redis).toBe(true); + }); + }); +}); diff --git a/tests/unit/processors/delayed-consumer-health.test.ts b/tests/unit/processors/delayed-consumer-health.test.ts new file mode 100644 index 0000000..4284dd4 --- /dev/null +++ b/tests/unit/processors/delayed-consumer-health.test.ts @@ -0,0 +1,92 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const eventListeners: Record void)[]> = {}; + +const kafkaConsumerMock = { + connect: vi.fn().mockResolvedValue(undefined), + subscribe: vi.fn().mockResolvedValue(undefined), + run: vi.fn().mockResolvedValue(undefined), + stop: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + commitOffsets: vi.fn().mockResolvedValue(undefined), + on: vi.fn().mockImplementation((event: string, listener: (e: unknown) => void) => { + if (!eventListeners[event]) { + eventListeners[event] = []; + } + eventListeners[event].push(listener); + }), + events: { + CRASH: 'consumer.crash', + HEARTBEAT: 'consumer.heartbeat', + STOP: 'consumer.stop', + DISCONNECT: 'consumer.disconnect', + CONNECT: 'consumer.connect', + }, +}; + +vi.mock('@src/config/kafka.config.js', () => ({ + kafka: { + consumer: vi.fn(() => kafkaConsumerMock), + }, +})); + +vi.mock('@src/processors/delayed/delayed.queue.js', () => ({ + addToDelayedQueue: vi.fn().mockResolvedValue(undefined), +})); + +describe('Delayed Consumer Health Checks', () => { + beforeEach(() => { + vi.clearAllMocks(); + for (const key of Object.keys(eventListeners)) { + delete eventListeners[key]; + } + }); + + it('should report unhealthy when consumer is not started', async () => { + const { isDelayedConsumerHealthy } = await import('@src/processors/delayed/delayed.consumer.js'); + expect(isDelayedConsumerHealthy()).toBe(false); + }); + + it('should report healthy once consumer starts', async () => { + const { startDelayedConsumer, isDelayedConsumerHealthy, stopDelayedConsumer } = await import( + '@src/processors/delayed/delayed.consumer.js' + ); + + await startDelayedConsumer(); + expect(isDelayedConsumerHealthy()).toBe(true); + + await stopDelayedConsumer(); + expect(isDelayedConsumerHealthy()).toBe(false); + }); + + it('should report unhealthy if consumer crashes', async () => { + const { startDelayedConsumer, isDelayedConsumerHealthy, stopDelayedConsumer } = await import( + '@src/processors/delayed/delayed.consumer.js' + ); + + await startDelayedConsumer(); + expect(isDelayedConsumerHealthy()).toBe(true); + + // Emit crash event + const crashHandlers = eventListeners['consumer.crash'] || []; + for (const handler of crashHandlers) { + handler({ payload: { error: new Error('Broker disconnected') } }); + } + + expect(isDelayedConsumerHealthy()).toBe(false); + + await stopDelayedConsumer(); + }); + + it('should report unhealthy if heartbeat is stale', async () => { + const { startDelayedConsumer, isDelayedConsumerHealthy, stopDelayedConsumer } = await import( + '@src/processors/delayed/delayed.consumer.js' + ); + + await startDelayedConsumer(); + // With negative allowance, immediately considered stale + expect(isDelayedConsumerHealthy(-1)).toBe(false); + + await stopDelayedConsumer(); + }); +}); diff --git a/tests/unit/utils/k8s-health-probe.test.ts b/tests/unit/utils/k8s-health-probe.test.ts new file mode 100644 index 0000000..818db60 --- /dev/null +++ b/tests/unit/utils/k8s-health-probe.test.ts @@ -0,0 +1,155 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import axios from 'axios'; +import { runHealthChecks, createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeCheck, HealthProbeServer } from '@src/types/types.js'; + +describe('k8s-health-probe utility', () => { + describe('runHealthChecks', () => { + it('should return ok: true when all checks pass', async () => { + const checks: HealthProbeCheck[] = [ + { name: 'db', check: () => true }, + { name: 'cache', check: async () => true }, + ]; + + const result = await runHealthChecks(checks); + expect(result.ok).toBe(true); + expect(result.results).toEqual({ + db: true, + cache: true, + }); + }); + + it('should return ok: false when one check fails', async () => { + const checks: HealthProbeCheck[] = [ + { name: 'db', check: () => true }, + { name: 'cache', check: () => false }, + ]; + + const result = await runHealthChecks(checks); + expect(result.ok).toBe(false); + expect(result.results).toEqual({ + db: true, + cache: false, + }); + }); + + it('should handle thrown errors gracefully and mark check false', async () => { + const checks: HealthProbeCheck[] = [ + { name: 'db', check: () => true }, + { + name: 'cache', + check: () => { + throw new Error('Connection refused'); + }, + }, + ]; + + const result = await runHealthChecks(checks); + expect(result.ok).toBe(false); + expect(result.results).toEqual({ + db: true, + cache: false, + }); + }); + + it('should handle empty checks gracefully', async () => { + const result = await runHealthChecks([]); + expect(result.ok).toBe(true); + expect(result.results).toEqual({}); + }); + }); + + describe('createHealthProbeServer', () => { + let probeServer: HealthProbeServer | null = null; + const testPort = 18991; + + afterEach(async () => { + if (probeServer) { + await probeServer.stop(); + probeServer = null; + } + }); + + it('should respond to /healthz and /readyz correctly when healthy', async () => { + const port = 18991; + probeServer = createHealthProbeServer({ + port, + serviceName: 'test-service', + livenessChecks: [{ name: 'process', check: () => true }], + readinessChecks: [{ name: 'db', check: () => true }], + }); + await probeServer.start(); + + const healthzRes = await axios.get(`http://127.0.0.1:${port}/healthz`); + expect(healthzRes.status).toBe(200); + expect(healthzRes.data).toEqual( + expect.objectContaining({ + service: 'test-service', + status: 'healthy', + checks: { process: true }, + }) + ); + + const readyzRes = await axios.get(`http://127.0.0.1:${port}/readyz`); + expect(readyzRes.status).toBe(200); + expect(readyzRes.data).toEqual( + expect.objectContaining({ + service: 'test-service', + status: 'ready', + checks: { db: true }, + }) + ); + }); + + it('should return 503 on /readyz when a readiness check fails', async () => { + const port = 18992; + probeServer = createHealthProbeServer({ + port, + serviceName: 'test-service', + readinessChecks: [ + { name: 'db', check: () => true }, + { name: 'redis', check: () => false }, + ], + }); + await probeServer.start(); + + try { + await axios.get(`http://127.0.0.1:${port}/readyz`); + expect.unreachable('Should have thrown HTTP 503'); + } catch (err: unknown) { + if (axios.isAxiosError(err)) { + expect(err.response?.status).toBe(503); + expect(err.response?.data).toEqual( + expect.objectContaining({ + service: 'test-service', + status: 'not_ready', + checks: { db: true, redis: false }, + }) + ); + } else { + throw err; + } + } + }); + + it('should return 404 for unknown endpoints', async () => { + const port = 18993; + probeServer = createHealthProbeServer({ + port, + serviceName: 'test-service', + }); + await probeServer.start(); + + try { + await axios.get(`http://127.0.0.1:${port}/unknown`); + expect.unreachable('Should have thrown HTTP 404'); + } catch (err: unknown) { + if (axios.isAxiosError(err)) { + expect(err.response?.status).toBe(404); + } else { + throw err; + } + } + }); + }); +}); diff --git a/tests/unit/workers/status-consumer-health.test.ts b/tests/unit/workers/status-consumer-health.test.ts new file mode 100644 index 0000000..ce71387 --- /dev/null +++ b/tests/unit/workers/status-consumer-health.test.ts @@ -0,0 +1,94 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const eventListeners: Record void)[]> = {}; + +const kafkaConsumerMock = { + connect: vi.fn().mockResolvedValue(undefined), + subscribe: vi.fn().mockResolvedValue(undefined), + run: vi.fn().mockResolvedValue(undefined), + stop: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + commitOffsets: vi.fn().mockResolvedValue(undefined), + on: vi.fn().mockImplementation((event: string, listener: (e: unknown) => void) => { + if (!eventListeners[event]) { + eventListeners[event] = []; + } + eventListeners[event].push(listener); + }), + events: { + CRASH: 'consumer.crash', + HEARTBEAT: 'consumer.heartbeat', + STOP: 'consumer.stop', + DISCONNECT: 'consumer.disconnect', + CONNECT: 'consumer.connect', + }, +}; + +vi.mock('@src/config/kafka.config.js', () => ({ + kafka: { + consumer: vi.fn(() => kafkaConsumerMock), + }, +})); + +vi.mock('@src/database/models/notification.models.js', () => ({ + default: { + findByIdAndUpdate: vi.fn(), + }, +})); + +describe('Status Consumer Health Checks', () => { + beforeEach(() => { + vi.clearAllMocks(); + for (const key of Object.keys(eventListeners)) { + delete eventListeners[key]; + } + }); + + it('should report unhealthy when consumer is not started', async () => { + const { isStatusConsumerHealthy } = await import('@src/workers/consumers/status.consumer.js'); + expect(isStatusConsumerHealthy()).toBe(false); + }); + + it('should report healthy once consumer starts', async () => { + const { startStatusConsumer, isStatusConsumerHealthy, stopStatusConsumer } = await import( + '@src/workers/consumers/status.consumer.js' + ); + + await startStatusConsumer(); + expect(isStatusConsumerHealthy()).toBe(true); + + await stopStatusConsumer(); + expect(isStatusConsumerHealthy()).toBe(false); + }); + + it('should report unhealthy if consumer crashes', async () => { + const { startStatusConsumer, isStatusConsumerHealthy, stopStatusConsumer } = await import( + '@src/workers/consumers/status.consumer.js' + ); + + await startStatusConsumer(); + expect(isStatusConsumerHealthy()).toBe(true); + + // Emit crash event + const crashHandlers = eventListeners['consumer.crash'] || []; + for (const handler of crashHandlers) { + handler({ payload: { error: new Error('Broker disconnected') } }); + } + + expect(isStatusConsumerHealthy()).toBe(false); + + await stopStatusConsumer(); + }); + + it('should report unhealthy if heartbeat is stale', async () => { + const { startStatusConsumer, isStatusConsumerHealthy, stopStatusConsumer } = await import( + '@src/workers/consumers/status.consumer.js' + ); + + await startStatusConsumer(); + // With threshold of 0ms, heartbeat is immediately considered stale if checked with negative/0 allowance + expect(isStatusConsumerHealthy(-1)).toBe(false); + + await stopStatusConsumer(); + }); +}); From 227312cb9a52b515c3690182402ebac958916b42 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Fri, 25 Sep 2026 18:32:03 +0530 Subject: [PATCH 32/46] docs(api-docs): update the api documentation --- api-docs/.env.example | 11 +- api-docs/README.md | 202 +++++++++++++++++- api-docs/admin_auth/Admin Login.bru | 23 ++ .../admin_auth/Check Admin Auth Status.bru | 16 ++ api-docs/admin_auth/Logout Admin Account.bru | 16 ++ api-docs/admin_auth/Setup Admin Account.bru | 23 ++ api-docs/admin_auth/folder.bru | 7 + api-docs/api_keys/Create API Key.bru | 24 +++ api-docs/api_keys/Delete API Key.bru | 16 ++ api-docs/api_keys/Get API Key Usage.bru | 16 ++ api-docs/api_keys/List API Keys.bru | 16 ++ api-docs/api_keys/Revoke API Key.bru | 16 ++ api-docs/api_keys/folder.bru | 7 + .../Delete Channel Routing.bru | 16 ++ .../channel_routing/Get Channel Routing.bru | 16 ++ .../channel_routing/List Channel Routings.bru | 16 ++ .../channel_routing/Set Channel Routing.bru | 24 +++ api-docs/channel_routing/folder.bru | 7 + api-docs/collection.bru | 2 +- api-docs/plugins/Change Plugin Version.bru | 23 ++ api-docs/plugins/Delete npm Auth Config.bru | 21 ++ api-docs/plugins/Get npm Auth Status.bru | 16 ++ api-docs/plugins/Install Plugin.bru | 28 +++ api-docs/plugins/List Installed Plugins.bru | 16 ++ api-docs/plugins/List Plugin Catalog.bru | 16 ++ api-docs/plugins/Save npm Auth Config.bru | 23 ++ api-docs/plugins/Uninstall Plugin.bru | 16 ++ api-docs/providers/Create Provider.bru | 35 +++ api-docs/providers/Delete Provider.bru | 16 ++ .../providers/Get Provider Rate Limit.bru | 16 ++ api-docs/providers/Get Provider by Id.bru | 20 ++ .../providers/Get Providers Rate Limits.bru | 16 ++ api-docs/providers/List Providers.bru | 16 ++ .../providers/Reset Provider Rate Limit.bru | 16 ++ .../providers/Test Provider Connection.bru | 26 +++ api-docs/providers/Update Provider.bru | 30 +++ api-docs/providers/folder.bru | 7 + .../settings/Get Operational Settings.bru | 16 ++ .../settings/Reset Operational Settings.bru | 16 ++ .../settings/Update Operational Settings.bru | 55 +++++ api-docs/settings/folder.bru | 7 + 41 files changed, 911 insertions(+), 14 deletions(-) create mode 100644 api-docs/admin_auth/Admin Login.bru create mode 100644 api-docs/admin_auth/Check Admin Auth Status.bru create mode 100644 api-docs/admin_auth/Logout Admin Account.bru create mode 100644 api-docs/admin_auth/Setup Admin Account.bru create mode 100644 api-docs/admin_auth/folder.bru create mode 100644 api-docs/api_keys/Create API Key.bru create mode 100644 api-docs/api_keys/Delete API Key.bru create mode 100644 api-docs/api_keys/Get API Key Usage.bru create mode 100644 api-docs/api_keys/List API Keys.bru create mode 100644 api-docs/api_keys/Revoke API Key.bru create mode 100644 api-docs/api_keys/folder.bru create mode 100644 api-docs/channel_routing/Delete Channel Routing.bru create mode 100644 api-docs/channel_routing/Get Channel Routing.bru create mode 100644 api-docs/channel_routing/List Channel Routings.bru create mode 100644 api-docs/channel_routing/Set Channel Routing.bru create mode 100644 api-docs/channel_routing/folder.bru create mode 100644 api-docs/plugins/Change Plugin Version.bru create mode 100644 api-docs/plugins/Delete npm Auth Config.bru create mode 100644 api-docs/plugins/Get npm Auth Status.bru create mode 100644 api-docs/plugins/Install Plugin.bru create mode 100644 api-docs/plugins/List Installed Plugins.bru create mode 100644 api-docs/plugins/List Plugin Catalog.bru create mode 100644 api-docs/plugins/Save npm Auth Config.bru create mode 100644 api-docs/plugins/Uninstall Plugin.bru create mode 100644 api-docs/providers/Create Provider.bru create mode 100644 api-docs/providers/Delete Provider.bru create mode 100644 api-docs/providers/Get Provider Rate Limit.bru create mode 100644 api-docs/providers/Get Provider by Id.bru create mode 100644 api-docs/providers/Get Providers Rate Limits.bru create mode 100644 api-docs/providers/List Providers.bru create mode 100644 api-docs/providers/Reset Provider Rate Limit.bru create mode 100644 api-docs/providers/Test Provider Connection.bru create mode 100644 api-docs/providers/Update Provider.bru create mode 100644 api-docs/providers/folder.bru create mode 100644 api-docs/settings/Get Operational Settings.bru create mode 100644 api-docs/settings/Reset Operational Settings.bru create mode 100644 api-docs/settings/Update Operational Settings.bru create mode 100644 api-docs/settings/folder.bru diff --git a/api-docs/.env.example b/api-docs/.env.example index 3fb5e47..12085d7 100644 --- a/api-docs/.env.example +++ b/api-docs/.env.example @@ -1,5 +1,6 @@ BASE_URL=http://localhost:3000 -NS_API_KEY= +API_KEY= +API_KEY_ID= TO_EMAIL= TO_EMAIL_2= CHANNEL=mock @@ -11,4 +12,10 @@ ALERT_ID= ADMIN_CHANNEL_ID= DISCORD_WEBHOOK_URL= TELEGRAM_BOT_TOKEN= -TELEGRAM_CHAT_ID= \ No newline at end of file +TELEGRAM_CHAT_ID= +ADMIN_USERNAME=admin +ADMIN_PASSWORD=SecureAdminPassword123! +PROVIDER_ID=mock-provider-1 +ROUTING_CHANNEL=mock +NPM_AUTH_TOKEN= +NPM_REGISTRY_URL=https://registry.npmjs.org/ \ No newline at end of file diff --git a/api-docs/README.md b/api-docs/README.md index e0798e6..4be9e96 100644 --- a/api-docs/README.md +++ b/api-docs/README.md @@ -24,12 +24,13 @@ Refer to the [Bruno Secrets Management & DotEnv Documentation](https://docs.useb | Variable | Description | Example / Default | | :--- | :--- | :--- | | `BASE_URL` | Base HTTP URL of the SimpleNS API server | `http://localhost:3000` | -| `NS_API_KEY` | SimpleNS API Bearer Key for authentication | `ns_...` | +| `API_KEY` | SimpleNS API Bearer Key for authentication | `sns_...` | +| `API_KEY_ID` | MongoDB ObjectId or key_id of an API key | `64f...` | | `TO_EMAIL` | Sample recipient email address | `user@example.com` | | `TO_EMAIL_2` | Secondary recipient email address for batch requests | `user2@example.com` | | `CHANNEL` | Notification channel identifier | `mock` | | `PROVIDER` | Channel provider identifier | `mock` | -| `PACKAGE_NAME` | Plugin package name for templates | `simplens-plugin-mock` | +| `PACKAGE_NAME` | Plugin package name for templates and plugins | `simplens-plugin-mock` | | `TEMPLATE_ID` | Identifier of a notification template | `mock-sample-1` | | `NOTIFICATION_ID` | MongoDB ObjectId of a notification for management endpoints | `64f...` | | `ALERT_ID` | MongoDB ObjectId of an alert for alert endpoints | `64f...` | @@ -37,6 +38,32 @@ Refer to the [Bruno Secrets Management & DotEnv Documentation](https://docs.useb | `DISCORD_WEBHOOK_URL` | Discord webhook URL for testing admin channel alerts | `https://discord.com/api/webhooks/...` | | `TELEGRAM_BOT_TOKEN` | Telegram Bot token for testing admin channel alerts | `123456789:ABCdefGhI...` | | `TELEGRAM_CHAT_ID` | Telegram chat ID for testing admin channel alerts | `-1001234567890` | +| `ADMIN_USERNAME` | Administrator username for initial setup & verification | `admin` | +| `ADMIN_PASSWORD` | Administrator password for initial setup & verification | `SecureAdminPassword123!` | +| `PROVIDER_ID` | Identifier for configured provider instance | `mock-provider-1` | +| `ROUTING_CHANNEL` | Notification channel identifier for routing config | `mock` | +| `NPM_AUTH_TOKEN` | Private npm registry authentication token | `npm_...` | +| `NPM_REGISTRY_URL` | Custom or default npm registry URL | `https://registry.npmjs.org/` | + +--- + +## Authentication & Authorization + +SimpleNS supports dual authentication across all protected API routes: + +1. **Admin Dashboard (Browser Cookie)**: + - Authenticated using the HTTP-only `simplens_session` cookie. + - Automatically set by the Core API when calling `POST /api/admin/auth/signup` or `POST /api/admin/auth/login`. + - Cleared on `POST /api/admin/auth/logout`. + - Browsers automatically attach this cookie on every request without requiring client-side token storage. + +2. **Programmatic API Clients & Services (API Key or Bearer Token)**: + - Authenticated using dynamic API keys (`sns_live_...` or `sns_test_...`) created via the Dashboard or API. + - Passed via either header: + - `Authorization: Bearer ` + - `x-api-key: ` + +All protected routes in SimpleNS accept either authentication method interchangeably. --- @@ -47,13 +74,32 @@ Public server and health information (no authentication required). - **API Root**: `GET /api` - Basic server info. - **Health Check**: `GET /api/health` - Health status and timestamp for Docker/Kubernetes probes. -### 2. `notifications` (Protected) +### 2. `admin_auth` (Public) +Administrator setup, authentication, and session management. +- **Check Admin Auth Status**: `GET /api/admin/auth/status` - Check whether an administrator account has already been configured. +- **Signup Admin Account**: `POST /api/admin/auth/signup` - Configure the primary administrator account (only allowed once during setup). Sets `simplens_session` cookie. + ```json + { + "username": "{{process.env.ADMIN_USERNAME}}", + "password": "{{process.env.ADMIN_PASSWORD}}" + } + ``` +- **Login Admin Account**: `POST /api/admin/auth/login` - Verify administrator credentials and set the `simplens_session` HTTP-only cookie. + ```json + { + "username": "{{process.env.ADMIN_USERNAME}}", + "password": "{{process.env.ADMIN_PASSWORD}}" + } + ``` +- **Logout Admin Account**: `POST /api/admin/auth/logout` - Clear and invalidate the `simplens_session` cookie. + +### 3. `notifications` (Protected) Notification dispatch and status. - **Single Request**: `POST /api/notification` - Enqueue a single notification across one or more channels. - **Batch Request**: `POST /api/notification/batch` - Enqueue batch notifications across recipients. - **Check Notification Endpoint**: `GET /api/notification` - Check notification service status. -### 3. `notifications_management` (Protected) +### 4. `notifications_management` (Protected) Query, inspect, retry, and delete notification records. - **List Notifications**: `GET /api/notifications` - Paginated, sorted, and filtered notification listing. - **Get Recent Notifications**: `GET /api/notifications/recent` - Live activity feed of latest notifications. @@ -61,7 +107,7 @@ Query, inspect, retry, and delete notification records. - **Retry Notification**: `POST /api/notifications/:id/retry` - Reset a failed notification to pending for re-execution. - **Delete Notification**: `DELETE /api/notifications/:id` - Delete a notification record. -### 4. `notification_templates` (Protected) +### 5. `notification_templates` (Protected) Manage reusable message templates for plugins. - **Create Template**: `POST /api/templates/create` - Register a new notification template. - **List All Templates**: `GET /api/templates` - Retrieve all templates. @@ -70,23 +116,98 @@ Manage reusable message templates for plugins. - **Update Template**: `PUT /api/templates/:template_id` - Update an existing template. - **Delete Template**: `DELETE /api/templates/:template_id` - Remove a template. -### 5. `plugins` (Protected) -Channel and provider discovery. +### 6. `plugins` (Protected) +Channel discovery, installed npm plugins, public plugin catalog, npm registry authentication, and plugin lifecycle. - **Get Plugin Metadata**: `GET /api/plugins` - List registered channels, active providers, and content/credential schemas. +- **List Installed Plugins**: `GET /api/plugins/installed` - List all installed npm plugins in MongoDB. +- **List Plugin Catalog**: `GET /api/plugins/catalog/:category` - Fetch official or community plugin catalog (`official` | `community`). +- **Get npm Auth Status**: `GET /api/plugins/npm-auth` - Retrieve npm authentication status with masked tokens. +- **Save npm Auth Config**: `POST /api/plugins/npm-auth` - Save custom npm token and registry configuration. + ```json + { + "token": "{{process.env.NPM_AUTH_TOKEN}}", + "registry_url": "{{process.env.NPM_REGISTRY_URL}}" + } + ``` +- **Delete npm Auth Config**: `DELETE /api/plugins/npm-auth` - Remove configured npm registry credentials. +- **Install Plugin**: `POST /api/plugins/install` - Download, verify, and register an npm plugin package. + ```json + { + "package": "{{process.env.PACKAGE_NAME}}", + "version": "latest", + "auth": { + "token": "{{process.env.NPM_AUTH_TOKEN}}", + "registry_url": "{{process.env.NPM_REGISTRY_URL}}", + "save_token": true + } + } + ``` +- **Change Plugin Version**: `PUT /api/plugins/version` - Upgrade or downgrade an installed plugin version. + ```json + { + "package": "{{process.env.PACKAGE_NAME}}", + "version": "1.0.1" + } + ``` +- **Uninstall Plugin**: `DELETE /api/plugins/:package` - Uninstall and clean up a plugin package. + +### 7. `providers` (Protected) +Manage provider instances, test connections, and inspect/reset token-bucket rate limits. +- **List Providers**: `GET /api/providers` - Retrieve all configured provider instances. +- **Create Provider**: `POST /api/providers` - Register a new provider with encrypted credentials and rate limiting options. + ```json + { + "id": "{{process.env.PROVIDER_ID}}", + "plugin_name": "{{process.env.PACKAGE_NAME}}", + "credentials": { + "apiKey": "sample-provider-api-key" + }, + "options": { + "priority": 1, + "rateLimit": { + "maxTokens": 100, + "refillRate": 10, + "refillInterval": "second" + } + }, + "enabled": true + } + ``` +- **Get Provider by Id**: `GET /api/providers/:id` - Fetch provider by ID (with optional `include_decrypted=true/false`). +- **Update Provider**: `PUT /api/providers/:id` - Update provider credentials, priority, or rate limits. +- **Delete Provider**: `DELETE /api/providers/:id` - Remove a provider instance. +- **Test Provider Connection**: `POST /api/providers/test` - Test provider connectivity without persisting credentials. +- **Get Providers Rate Limits**: `GET /api/providers/rate-limits` - Real-time rate limiter status across all providers. +- **Get Provider Rate Limit**: `GET /api/providers/:id/rate-limit` - Real-time rate limiter tokens and telemetry for a specific provider. +- **Reset Provider Rate Limit**: `POST /api/providers/:id/rate-limit/reset` - Manually refill tokens and clear rate limiter backoffs. -### 6. `alerts` (Protected) +### 8. `channel_routing` (Protected) +Configure default providers, fallback provider chains, and Kafka partition count per notification channel. +- **List Channel Routings**: `GET /api/channels/routing` - List all channel routing configurations. +- **Get Channel Routing**: `GET /api/channels/routing/:channel` - Retrieve routing for a specific channel (e.g. `email`, `mock`). +- **Set Channel Routing**: `PUT /api/channels/routing/:channel` - Configure default provider, cascading fallback array, and partitions. + ```json + { + "default_provider_id": "{{process.env.PROVIDER_ID}}", + "fallback_provider_ids": [], + "partitions": 6 + } + ``` +- **Delete Channel Routing**: `DELETE /api/channels/routing/:channel` - Remove routing configuration for a channel. + +### 9. `alerts` (Protected) Manage system alerts triggered by failures or stuck workers. - **List Alerts**: `GET /api/alerts` - List unresolved alerts with pagination and type summary counts. - **Resolve Alert**: `POST /api/alerts/:id/resolve` - Mark alert as resolved and retry the underlying notification. - **Bulk Resolve Alerts**: `POST /api/alerts/bulk-resolve` - Resolve multiple alerts and retry notifications in batch. - **Dismiss Alert**: `DELETE /api/alerts/:id` - Dismiss an alert without retrying. -### 7. `dashboard` (Protected) +### 10. `dashboard` (Protected) Analytics and trends for the admin dashboard. - **Get Dashboard Stats**: `GET /api/dashboard/stats` - Notification counts grouped by status and channel. - **Get Dashboard Trends**: `GET /api/dashboard/trends` - Hourly or daily time-series counts (`24h`, `7d`, `30d`). -### 8. `admin_channels` (Protected) +### 11. `admin_channels` (Protected) Manage administrator alert channels (Discord, Telegram, etc.). - **List Admin Channel Providers**: `GET /api/admin-channels/providers` - Available provider integrations and schemas. - **Validate Admin Channel Config**: `POST /api/admin-channels/validate` - Validate channel credentials against schema without saving. @@ -95,4 +216,63 @@ Manage administrator alert channels (Discord, Telegram, etc.). - **Create Admin Channel**: `POST /api/admin-channels` - Create a new encrypted admin alert channel. - **Get Admin Channel by Id**: `GET /api/admin-channels/:id` - Fetch single channel details. - **Update Admin Channel**: `PATCH /api/admin-channels/:id` - Update channel name, active status, filters, or config. -- **Delete Admin Channel**: `DELETE /api/admin-channels/:id` - Remove an alert channel. \ No newline at end of file +- **Delete Admin Channel**: `DELETE /api/admin-channels/:id` - Remove an alert channel. + +### 12. `settings` (Protected) +Read, modify, and reset live operational configuration across services via Redis sync. +- **Get Operational Settings**: `GET /api/settings` - Retrieve current active settings for API, workers, retry, delayed poller, recovery, and logging. +- **Update Operational Settings**: `PUT /api/settings` - Update one or more configuration fields. + ```json + { + "api": { + "max_batch_req_limit": 1000 + }, + "worker": { + "outbox_poll_interval_ms": 5000, + "outbox_cleanup_interval_ms": 60000, + "outbox_batch_size": 100, + "outbox_retention_ms": 300000, + "outbox_claim_timeout_ms": 30000 + }, + "retry": { + "max_retry_count": 5, + "idempotency_ttl_seconds": 86400, + "processing_ttl_seconds": 120, + "rate_limit_retry_delay_ms": 5000 + }, + "delayed": { + "delayed_poll_interval_ms": 1000, + "delayed_batch_size": 10, + "max_poller_retries": 3 + }, + "recovery": { + "recovery_poll_interval_ms": 60000, + "processing_stuck_threshold_ms": 300000, + "pending_stuck_threshold_ms": 300000, + "recovery_batch_size": 50, + "recovery_claim_timeout_ms": 60000, + "cleanup_resolved_alerts_retention_ms": 86400000, + "cleanup_processed_status_outbox_retention_ms": 86400000 + }, + "logging": { + "log_level": "info", + "log_to_file": true + } + } + ``` +- **Reset Operational Settings**: `POST /api/settings/reset` - Reset operational settings to system defaults and broadcast to all services. + +### 13. `api_keys` (Protected) +Create, manage, inspect, and revoke programmatic API keys. +- **List API Keys**: `GET /api/keys` - List all active and revoked API keys. +- **Create API Key**: `POST /api/keys` - Generate a new scoped API key (`sns_live_...` or `sns_test_...`). + ```json + { + "name": "Integration Service Key", + "type": "live", + "scopes": ["*"] + } + ``` +- **Get API Key Usage**: `GET /api/keys/:id` - Inspect key metadata, usage counters, and last used timestamp. +- **Revoke API Key**: `POST /api/keys/:id/revoke` - Immediately deactivate an active API key. +- **Delete API Key**: `DELETE /api/keys/:id` - Permanently remove an API key record. \ No newline at end of file diff --git a/api-docs/admin_auth/Admin Login.bru b/api-docs/admin_auth/Admin Login.bru new file mode 100644 index 0000000..a641099 --- /dev/null +++ b/api-docs/admin_auth/Admin Login.bru @@ -0,0 +1,23 @@ +meta { + name: Login Admin Account + type: http + seq: 3 +} + +post { + url: {{process.env.BASE_URL}}/api/admin/auth/login + body: json + auth: none +} + +body:json { + { + "username": "{{process.env.ADMIN_USERNAME}}", + "password": "{{process.env.ADMIN_PASSWORD}}" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/admin_auth/Check Admin Auth Status.bru b/api-docs/admin_auth/Check Admin Auth Status.bru new file mode 100644 index 0000000..66cd7ae --- /dev/null +++ b/api-docs/admin_auth/Check Admin Auth Status.bru @@ -0,0 +1,16 @@ +meta { + name: Check Admin Auth Status + type: http + seq: 1 +} + +get { + url: {{process.env.BASE_URL}}/api/admin/auth/status + body: none + auth: none +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/admin_auth/Logout Admin Account.bru b/api-docs/admin_auth/Logout Admin Account.bru new file mode 100644 index 0000000..b66b19a --- /dev/null +++ b/api-docs/admin_auth/Logout Admin Account.bru @@ -0,0 +1,16 @@ +meta { + name: Logout Admin Account + type: http + seq: 4 +} + +post { + url: {{process.env.BASE_URL}}/api/admin/auth/logout + body: none + auth: none +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/admin_auth/Setup Admin Account.bru b/api-docs/admin_auth/Setup Admin Account.bru new file mode 100644 index 0000000..eefc07f --- /dev/null +++ b/api-docs/admin_auth/Setup Admin Account.bru @@ -0,0 +1,23 @@ +meta { + name: Signup Admin Account + type: http + seq: 2 +} + +post { + url: {{process.env.BASE_URL}}/api/admin/auth/signup + body: json + auth: none +} + +body:json { + { + "username": "{{process.env.ADMIN_USERNAME}}", + "password": "{{process.env.ADMIN_PASSWORD}}" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/admin_auth/folder.bru b/api-docs/admin_auth/folder.bru new file mode 100644 index 0000000..eb85e74 --- /dev/null +++ b/api-docs/admin_auth/folder.bru @@ -0,0 +1,7 @@ +meta { + name: admin_auth +} + +auth { + mode: none +} diff --git a/api-docs/api_keys/Create API Key.bru b/api-docs/api_keys/Create API Key.bru new file mode 100644 index 0000000..9057919 --- /dev/null +++ b/api-docs/api_keys/Create API Key.bru @@ -0,0 +1,24 @@ +meta { + name: Create API Key + type: http + seq: 2 +} + +post { + url: {{process.env.BASE_URL}}/api/keys + body: json + auth: inherit +} + +body:json { + { + "name": "Integration Test Key", + "type": "live", + "scopes": ["*"] + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/api_keys/Delete API Key.bru b/api-docs/api_keys/Delete API Key.bru new file mode 100644 index 0000000..5d681ee --- /dev/null +++ b/api-docs/api_keys/Delete API Key.bru @@ -0,0 +1,16 @@ +meta { + name: Delete API Key + type: http + seq: 5 +} + +delete { + url: {{process.env.BASE_URL}}/api/keys/{{process.env.API_KEY_ID}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/api_keys/Get API Key Usage.bru b/api-docs/api_keys/Get API Key Usage.bru new file mode 100644 index 0000000..538928b --- /dev/null +++ b/api-docs/api_keys/Get API Key Usage.bru @@ -0,0 +1,16 @@ +meta { + name: Get API Key Usage + type: http + seq: 3 +} + +get { + url: {{process.env.BASE_URL}}/api/keys/{{process.env.API_KEY_ID}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/api_keys/List API Keys.bru b/api-docs/api_keys/List API Keys.bru new file mode 100644 index 0000000..6242315 --- /dev/null +++ b/api-docs/api_keys/List API Keys.bru @@ -0,0 +1,16 @@ +meta { + name: List API Keys + type: http + seq: 1 +} + +get { + url: {{process.env.BASE_URL}}/api/keys + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/api_keys/Revoke API Key.bru b/api-docs/api_keys/Revoke API Key.bru new file mode 100644 index 0000000..1b9885c --- /dev/null +++ b/api-docs/api_keys/Revoke API Key.bru @@ -0,0 +1,16 @@ +meta { + name: Revoke API Key + type: http + seq: 4 +} + +post { + url: {{process.env.BASE_URL}}/api/keys/{{process.env.API_KEY_ID}}/revoke + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/api_keys/folder.bru b/api-docs/api_keys/folder.bru new file mode 100644 index 0000000..d227949 --- /dev/null +++ b/api-docs/api_keys/folder.bru @@ -0,0 +1,7 @@ +meta { + name: api_keys +} + +auth { + mode: inherit +} diff --git a/api-docs/channel_routing/Delete Channel Routing.bru b/api-docs/channel_routing/Delete Channel Routing.bru new file mode 100644 index 0000000..2e906b1 --- /dev/null +++ b/api-docs/channel_routing/Delete Channel Routing.bru @@ -0,0 +1,16 @@ +meta { + name: Delete Channel Routing + type: http + seq: 4 +} + +delete { + url: {{process.env.BASE_URL}}/api/channels/routing/{{process.env.ROUTING_CHANNEL}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/channel_routing/Get Channel Routing.bru b/api-docs/channel_routing/Get Channel Routing.bru new file mode 100644 index 0000000..a63cb4a --- /dev/null +++ b/api-docs/channel_routing/Get Channel Routing.bru @@ -0,0 +1,16 @@ +meta { + name: Get Channel Routing + type: http + seq: 2 +} + +get { + url: {{process.env.BASE_URL}}/api/channels/routing/{{process.env.ROUTING_CHANNEL}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/channel_routing/List Channel Routings.bru b/api-docs/channel_routing/List Channel Routings.bru new file mode 100644 index 0000000..032d6d4 --- /dev/null +++ b/api-docs/channel_routing/List Channel Routings.bru @@ -0,0 +1,16 @@ +meta { + name: List Channel Routings + type: http + seq: 1 +} + +get { + url: {{process.env.BASE_URL}}/api/channels/routing + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/channel_routing/Set Channel Routing.bru b/api-docs/channel_routing/Set Channel Routing.bru new file mode 100644 index 0000000..98555a8 --- /dev/null +++ b/api-docs/channel_routing/Set Channel Routing.bru @@ -0,0 +1,24 @@ +meta { + name: Set Channel Routing + type: http + seq: 3 +} + +put { + url: {{process.env.BASE_URL}}/api/channels/routing/{{process.env.ROUTING_CHANNEL}} + body: json + auth: inherit +} + +body:json { + { + "default_provider_id": "{{process.env.PROVIDER_ID}}", + "fallback_provider_ids": [], + "partitions": 6 + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/channel_routing/folder.bru b/api-docs/channel_routing/folder.bru new file mode 100644 index 0000000..f6dd828 --- /dev/null +++ b/api-docs/channel_routing/folder.bru @@ -0,0 +1,7 @@ +meta { + name: channel_routing +} + +auth { + mode: inherit +} diff --git a/api-docs/collection.bru b/api-docs/collection.bru index 22ec557..776f23a 100644 --- a/api-docs/collection.bru +++ b/api-docs/collection.bru @@ -3,5 +3,5 @@ auth { } auth:bearer { - token: {{process.env.NS_API_KEY}} + token: {{process.env.API_KEY}} } diff --git a/api-docs/plugins/Change Plugin Version.bru b/api-docs/plugins/Change Plugin Version.bru new file mode 100644 index 0000000..a308a48 --- /dev/null +++ b/api-docs/plugins/Change Plugin Version.bru @@ -0,0 +1,23 @@ +meta { + name: Change Plugin Version + type: http + seq: 8 +} + +put { + url: {{process.env.BASE_URL}}/api/plugins/version + body: json + auth: inherit +} + +body:json { + { + "package": "{{process.env.PACKAGE_NAME}}", + "version": "1.0.1" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/Delete npm Auth Config.bru b/api-docs/plugins/Delete npm Auth Config.bru new file mode 100644 index 0000000..e0033cc --- /dev/null +++ b/api-docs/plugins/Delete npm Auth Config.bru @@ -0,0 +1,21 @@ +meta { + name: Delete npm Auth Config + type: http + seq: 6 +} + +delete { + url: {{process.env.BASE_URL}}/api/plugins/npm-auth?id= + body: none + auth: inherit +} + +params:query { + ~id: + ~scope: +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/Get npm Auth Status.bru b/api-docs/plugins/Get npm Auth Status.bru new file mode 100644 index 0000000..436ccc5 --- /dev/null +++ b/api-docs/plugins/Get npm Auth Status.bru @@ -0,0 +1,16 @@ +meta { + name: Get npm Auth Status + type: http + seq: 4 +} + +get { + url: {{process.env.BASE_URL}}/api/plugins/npm-auth + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/Install Plugin.bru b/api-docs/plugins/Install Plugin.bru new file mode 100644 index 0000000..06c396c --- /dev/null +++ b/api-docs/plugins/Install Plugin.bru @@ -0,0 +1,28 @@ +meta { + name: Install Plugin + type: http + seq: 7 +} + +post { + url: {{process.env.BASE_URL}}/api/plugins/install + body: json + auth: inherit +} + +body:json { + { + "package": "{{process.env.PACKAGE_NAME}}", + "version": "latest", + "auth": { + "token": "{{process.env.NPM_AUTH_TOKEN}}", + "registry_url": "{{process.env.NPM_REGISTRY_URL}}", + "save_token": true + } + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/List Installed Plugins.bru b/api-docs/plugins/List Installed Plugins.bru new file mode 100644 index 0000000..f606027 --- /dev/null +++ b/api-docs/plugins/List Installed Plugins.bru @@ -0,0 +1,16 @@ +meta { + name: List Installed Plugins + type: http + seq: 2 +} + +get { + url: {{process.env.BASE_URL}}/api/plugins/installed + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/List Plugin Catalog.bru b/api-docs/plugins/List Plugin Catalog.bru new file mode 100644 index 0000000..f29d90f --- /dev/null +++ b/api-docs/plugins/List Plugin Catalog.bru @@ -0,0 +1,16 @@ +meta { + name: List Plugin Catalog + type: http + seq: 3 +} + +get { + url: {{process.env.BASE_URL}}/api/plugins/catalog/official + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/Save npm Auth Config.bru b/api-docs/plugins/Save npm Auth Config.bru new file mode 100644 index 0000000..bb02139 --- /dev/null +++ b/api-docs/plugins/Save npm Auth Config.bru @@ -0,0 +1,23 @@ +meta { + name: Save npm Auth Config + type: http + seq: 5 +} + +post { + url: {{process.env.BASE_URL}}/api/plugins/npm-auth + body: json + auth: inherit +} + +body:json { + { + "token": "{{process.env.NPM_AUTH_TOKEN}}", + "registry_url": "{{process.env.NPM_REGISTRY_URL}}" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/Uninstall Plugin.bru b/api-docs/plugins/Uninstall Plugin.bru new file mode 100644 index 0000000..74498ea --- /dev/null +++ b/api-docs/plugins/Uninstall Plugin.bru @@ -0,0 +1,16 @@ +meta { + name: Uninstall Plugin + type: http + seq: 9 +} + +delete { + url: {{process.env.BASE_URL}}/api/plugins/{{process.env.PACKAGE_NAME}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Create Provider.bru b/api-docs/providers/Create Provider.bru new file mode 100644 index 0000000..6a51e66 --- /dev/null +++ b/api-docs/providers/Create Provider.bru @@ -0,0 +1,35 @@ +meta { + name: Create Provider + type: http + seq: 2 +} + +post { + url: {{process.env.BASE_URL}}/api/providers + body: json + auth: inherit +} + +body:json { + { + "id": "{{process.env.PROVIDER_ID}}", + "plugin_name": "{{process.env.PACKAGE_NAME}}", + "credentials": { + "apiKey": "sample-provider-api-key" + }, + "options": { + "priority": 1, + "rateLimit": { + "maxTokens": 100, + "refillRate": 10, + "refillInterval": "second" + } + }, + "enabled": true + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Delete Provider.bru b/api-docs/providers/Delete Provider.bru new file mode 100644 index 0000000..f894def --- /dev/null +++ b/api-docs/providers/Delete Provider.bru @@ -0,0 +1,16 @@ +meta { + name: Delete Provider + type: http + seq: 5 +} + +delete { + url: {{process.env.BASE_URL}}/api/providers/{{process.env.PROVIDER_ID}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Get Provider Rate Limit.bru b/api-docs/providers/Get Provider Rate Limit.bru new file mode 100644 index 0000000..2bc4f8c --- /dev/null +++ b/api-docs/providers/Get Provider Rate Limit.bru @@ -0,0 +1,16 @@ +meta { + name: Get Provider Rate Limit + type: http + seq: 8 +} + +get { + url: {{process.env.BASE_URL}}/api/providers/{{process.env.PROVIDER_ID}}/rate-limit + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Get Provider by Id.bru b/api-docs/providers/Get Provider by Id.bru new file mode 100644 index 0000000..66c1226 --- /dev/null +++ b/api-docs/providers/Get Provider by Id.bru @@ -0,0 +1,20 @@ +meta { + name: Get Provider by Id + type: http + seq: 3 +} + +get { + url: {{process.env.BASE_URL}}/api/providers/{{process.env.PROVIDER_ID}}?include_decrypted=false + body: none + auth: inherit +} + +params:query { + include_decrypted: false +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Get Providers Rate Limits.bru b/api-docs/providers/Get Providers Rate Limits.bru new file mode 100644 index 0000000..c83de8c --- /dev/null +++ b/api-docs/providers/Get Providers Rate Limits.bru @@ -0,0 +1,16 @@ +meta { + name: Get Providers Rate Limits + type: http + seq: 7 +} + +get { + url: {{process.env.BASE_URL}}/api/providers/rate-limits + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/List Providers.bru b/api-docs/providers/List Providers.bru new file mode 100644 index 0000000..1d3fcb9 --- /dev/null +++ b/api-docs/providers/List Providers.bru @@ -0,0 +1,16 @@ +meta { + name: List Providers + type: http + seq: 1 +} + +get { + url: {{process.env.BASE_URL}}/api/providers + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Reset Provider Rate Limit.bru b/api-docs/providers/Reset Provider Rate Limit.bru new file mode 100644 index 0000000..6df0533 --- /dev/null +++ b/api-docs/providers/Reset Provider Rate Limit.bru @@ -0,0 +1,16 @@ +meta { + name: Reset Provider Rate Limit + type: http + seq: 9 +} + +post { + url: {{process.env.BASE_URL}}/api/providers/{{process.env.PROVIDER_ID}}/rate-limit/reset + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Test Provider Connection.bru b/api-docs/providers/Test Provider Connection.bru new file mode 100644 index 0000000..6be0172 --- /dev/null +++ b/api-docs/providers/Test Provider Connection.bru @@ -0,0 +1,26 @@ +meta { + name: Test Provider Connection + type: http + seq: 6 +} + +post { + url: {{process.env.BASE_URL}}/api/providers/test + body: json + auth: inherit +} + +body:json { + { + "plugin_name": "{{process.env.PACKAGE_NAME}}", + "credentials": { + "apiKey": "sample-provider-api-key" + }, + "options": {} + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Update Provider.bru b/api-docs/providers/Update Provider.bru new file mode 100644 index 0000000..a8e8065 --- /dev/null +++ b/api-docs/providers/Update Provider.bru @@ -0,0 +1,30 @@ +meta { + name: Update Provider + type: http + seq: 4 +} + +put { + url: {{process.env.BASE_URL}}/api/providers/{{process.env.PROVIDER_ID}} + body: json + auth: inherit +} + +body:json { + { + "enabled": true, + "options": { + "priority": 2, + "rateLimit": { + "maxTokens": 200, + "refillRate": 20, + "refillInterval": "second" + } + } + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/folder.bru b/api-docs/providers/folder.bru new file mode 100644 index 0000000..10dac8b --- /dev/null +++ b/api-docs/providers/folder.bru @@ -0,0 +1,7 @@ +meta { + name: providers +} + +auth { + mode: inherit +} diff --git a/api-docs/settings/Get Operational Settings.bru b/api-docs/settings/Get Operational Settings.bru new file mode 100644 index 0000000..ebd5cd2 --- /dev/null +++ b/api-docs/settings/Get Operational Settings.bru @@ -0,0 +1,16 @@ +meta { + name: Get Operational Settings + type: http + seq: 1 +} + +get { + url: {{process.env.BASE_URL}}/api/settings + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/settings/Reset Operational Settings.bru b/api-docs/settings/Reset Operational Settings.bru new file mode 100644 index 0000000..7f64191 --- /dev/null +++ b/api-docs/settings/Reset Operational Settings.bru @@ -0,0 +1,16 @@ +meta { + name: Reset Operational Settings + type: http + seq: 3 +} + +post { + url: {{process.env.BASE_URL}}/api/settings/reset + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/settings/Update Operational Settings.bru b/api-docs/settings/Update Operational Settings.bru new file mode 100644 index 0000000..59491b0 --- /dev/null +++ b/api-docs/settings/Update Operational Settings.bru @@ -0,0 +1,55 @@ +meta { + name: Update Operational Settings + type: http + seq: 2 +} + +put { + url: {{process.env.BASE_URL}}/api/settings + body: json + auth: inherit +} + +body:json { + { + "api": { + "max_batch_req_limit": 1000 + }, + "worker": { + "outbox_poll_interval_ms": 5000, + "outbox_cleanup_interval_ms": 60000, + "outbox_batch_size": 100, + "outbox_retention_ms": 300000, + "outbox_claim_timeout_ms": 30000 + }, + "retry": { + "max_retry_count": 5, + "idempotency_ttl_seconds": 86400, + "processing_ttl_seconds": 120, + "rate_limit_retry_delay_ms": 5000 + }, + "delayed": { + "delayed_poll_interval_ms": 1000, + "delayed_batch_size": 10, + "max_poller_retries": 3 + }, + "recovery": { + "recovery_poll_interval_ms": 60000, + "processing_stuck_threshold_ms": 300000, + "pending_stuck_threshold_ms": 300000, + "recovery_batch_size": 50, + "recovery_claim_timeout_ms": 60000, + "cleanup_resolved_alerts_retention_ms": 86400000, + "cleanup_processed_status_outbox_retention_ms": 86400000 + }, + "logging": { + "log_level": "info", + "log_to_file": true + } + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/settings/folder.bru b/api-docs/settings/folder.bru new file mode 100644 index 0000000..c24c666 --- /dev/null +++ b/api-docs/settings/folder.bru @@ -0,0 +1,7 @@ +meta { + name: settings +} + +auth { + mode: inherit +} From 3dd1ebd3a73d158fab4874fcad199c74bf733cdb Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Fri, 25 Sep 2026 18:50:45 +0530 Subject: [PATCH 33/46] docs: enhance README with MCP documentation and plugin management details --- README.md | 85 +++++++++++-------------------------------------------- 1 file changed, 17 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 8de8470..1981999 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ npx -y @simlpens/mcp --stdio ``` *(Supports both `streamable-http` and `stdio` transports)* +Read the [documentation here](https://www.simplens.in/docs/mcp/getting-started) to get started with SimpleNS MCP + --- ## Architecture @@ -110,26 +112,11 @@ SimpleNS is built around a strictly **decoupled, API-first model**: ### Plugin System -SimpleNS Core handles **orchestration**; plugins handle **delivery**. - -Plugins are automatically installed at runtime based on your `simplens.config.yaml` configuration. Use the **Config Generator CLI** to create or update your config: - -```bash -# Generate config for a plugin -npx @simplens/config-gen generate @simplens/mock - -# Generate config for multiple plugins -npx @simplens/config-gen gen @simplens/nodemailer-gmail @simplens/twilio-sms +SimpleNS Core handles **orchestration**, plugins handle **delivery**. -# Add a plugin to existing config -npx @simplens/config-gen gen @simplens/nodemailer-gmail -c simplens.config.yaml +Plugins can be installed directly from the dashboard, via API and MCP server -# List available official plugins -npx @simplens/config-gen list --offical -#List available community plugins -npx @simplens/config-gen list --community -``` #### Building Custom Plugins @@ -207,7 +194,7 @@ Expected response: **Using the Dashboard (recommended):** 1. Open [http://localhost:3002](http://localhost:3002) -2. Login with your configured credentials (default: `admin` / ``) +2. At startup, it lets you to create an admin account, install official and community plugins, setup their corresponding providers and credentials. Then go to dashboard home and create an API key for the external services to access the SimpleNS API server. 3. Navigate to **Send**, select a channel/provider, fill in the fields, and click **Send Notification** **Using cURL:** @@ -215,7 +202,7 @@ Expected response: ```bash curl -X POST http://localhost:3000/api/notification \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer " \ + -H "Authorization: Bearer " \ -d '{ "request_id": "550e8400-e29b-41d4-a716-446655440000", "client_id": "55283667-1f58-467d-86a1-47f7f0e059f2", @@ -246,17 +233,8 @@ curl -X POST http://localhost:3000/api/notification \ | API Server | [http://localhost:3000](http://localhost:3000) | Notification API | | Admin Dashboard | [http://localhost:3002](http://localhost:3002) | Monitoring & management | | Grafana | [http://localhost:3001](http://localhost:3001) | Log visualization | -| Kafka UI | [http://localhost:8080](http://localhost:8080) | Kafka topic monitoring | -### Add More Plugins -```bash -# Add Gmail email provider -npx @simplens/config-gen generate @simplens/nodemailer-gmail -c simplens.config.yaml - -# List available plugins -npx @simplens/config-gen list --official -``` > For production deployments, see the [Self-Hosting Guide](https://simplens.in/docs/core/self-hosting) which covers distributed deployments, cloud infrastructure integration, and security hardening. @@ -266,35 +244,6 @@ npx @simplens/config-gen list --official SimpleNS provides a comprehensive, authenticated REST API that powers both external clients and the Admin Dashboard. Because the core engine is decoupled from the UI, every action possible in the dashboard can be automated or integrated via REST endpoints. -| Category | Route | Method | Description | -|---|---|---|---| -| **System** | `/api` | `GET` | API Server status | -| | `/api/health` | `GET` | Health check probe (Docker / K8s) | -| **Notifications** | `/api/notification` | `POST` | Dispatch single notification | -| | `/api/notification/batch` | `POST` | Dispatch batch notification | -| | `/api/notification` | `GET` | Notification endpoint health | -| **Management** | `/api/notifications` | `GET` | Search & filter notification history | -| | `/api/notifications/recent` | `GET` | Activity feed of latest notifications | -| | `/api/notifications/:id` | `GET` | Fetch single notification details | -| | `/api/notifications/:id/retry` | `POST` | Reset failed notification for retry | -| | `/api/notifications/:id` | `DELETE` | Delete notification record | -| **Templates** | `/api/templates/create` | `POST` | Create reusable notification template | -| | `/api/templates` | `GET` | List all templates (or filter by package) | -| | `/api/templates/:template_id` | `GET` | Fetch template by ID | -| | `/api/templates/:template_id` | `PUT` | Update notification template | -| | `/api/templates/:template_id` | `DELETE` | Delete notification template | -| **Plugins** | `/api/plugins` | `GET` | Dynamic plugin metadata & schemas | -| **Alerts** | `/api/alerts` | `GET` | List system alerts | -| | `/api/alerts/:id/resolve` | `POST` | Resolve alert and retry notification | -| | `/api/alerts/bulk-resolve` | `POST` | Bulk resolve alerts and retry | -| | `/api/alerts/:id` | `DELETE` | Dismiss alert without retrying | -| **Dashboard** | `/api/dashboard/stats` | `GET` | Notification counts by status & channel | -| | `/api/dashboard/trends` | `GET` | Historical time-series trend data | -| **Admin Channels** | `/api/admin-channels/providers` | `GET` | Available alerting channel providers | -| | `/api/admin-channels/validate` | `POST` | Validate channel configuration schema | -| | `/api/admin-channels/test` | `POST` | Test live webhook or bot connection | -| | `/api/admin-channels` | `GET` / `POST` | List or create admin alert channels | -| | `/api/admin-channels/:id` | `GET` / `PATCH` / `DELETE` | Manage individual admin alert channels | > 📖 **Interactive API Collection**: A complete [Bruno](https://www.usebruno.com/) collection is available in [`/api-docs`](./api-docs) with preconfigured environments and sample requests for all endpoints. @@ -311,17 +260,17 @@ The Admin Dashboard is an independent Next.js application built strictly as a ** - 🏢 **Headless Ready**: Because the dashboard is fully decoupled, you can operate SimpleNS headless in backend environments, deploy custom frontends, or interact via CI/CD and AI tools without running the dashboard. **Key Dashboard Features:** -- 🏠 **Dashboard Home** — Status overview powered by `/api/dashboard/stats` -- 📡 **Channel Cards** — Visual status for each configured delivery channel -- 📋 **Events Explorer** — Paginated event table with live filtering via `/api/notifications` -- 🔴 **Failed Events** — Dedicated view with one-click retries via `/api/notifications/:id/retry` -- 🚨 **Alerts** — System health and orphaned recovery alerts via `/api/alerts` -- 📈 **Analytics** — 24h, 7d, and 30d time-series trends via `/api/dashboard/trends` -- 🔌 **Plugins** — Dynamic form generation based on runtime schemas from `/api/plugins` -- 🔧 **Payload Studio** — Interactive schema explorer for constructing valid payloads -- 🔔 **Admin Alert Channels** — Live test and configure Discord/Telegram alerts via `/api/admin-channels` -- 📝 **Notification Templates** — Full lifecycle management and live preview via `/api/templates` - +- **Dashboard Home** — Status overview powered by `/api/dashboard/stats` +- **Channel Cards** — Visual status for each configured delivery channel +- **Events Explorer** — Paginated event table with live filtering via `/api/notifications` +- **Failed Events** — Dedicated view with one-click retries via `/api/notifications/:id/retry` +- **Alerts** — System health and orphaned recovery alerts via `/api/alerts` +- **Analytics** — 24h, 7d, and 30d time-series trends via `/api/dashboard/trends` +- **Plugins** — Manage official, community and custom plugins, setup providers with credentials and setup channel routing with an array of fallback providers +- **Payload Studio** — Interactive schema explorer for constructing valid payloads +- **Admin Alert Channels** — Live test and configure Discord/Telegram alerts via `/api/admin-channels` +- **Notification Templates** — Full lifecycle management and live preview via `/api/templates` +- **API Key** - Create and manage multiple API keys, track their usage and so on --- ## License From 379547c3017b6eaadbd2f6613487f88bc57a2c67 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 27 Sep 2026 18:50:57 +0530 Subject: [PATCH 34/46] feat(settings): add new recovery settings for claim timeout and alert retention feat(api-keys): introduce apiKeyType for notification and admin keys fix(docker): remove insecure default JWT_SECRET in docker-compose files feat(plugins): enhance plugin management with version change and uninstall validation fix(auth): restrict notification API keys from accessing management routes feat(jwt): implement JWT_SECRET validation for security during server startup feat(api-key): update API key model to include type and enforce validation test(api-keys): add tests for API key management access based on key type test(auth): enhance middleware tests for API key type restrictions test(plugin-fs): add unit tests for npm package name and version validation --- .env.example | 2 +- dashboard/app/keys/page.tsx | 76 ++++++++++++--- dashboard/app/settings/page.tsx | 59 ++++++++++++ dashboard/lib/types.ts | 4 + docker-compose/docker-compose.dev.yaml | 2 +- docker-compose/docker-compose.local.yaml | 2 +- docker-compose/docker-compose.master.yaml | 2 +- src/api/controllers/api-key.controller.ts | 4 +- src/api/controllers/plugins.controller.ts | 32 +++++-- src/api/middlewares/auth_middleware.ts | 23 ++++- src/api/server.ts | 3 + src/config/env.config.ts | 34 ++++++- src/database/models/api-key.models.ts | 6 ++ src/plugins/loader/plugin-fs.ts | 70 +++++++++----- src/plugins/loader/registry.ts | 30 +++++- .../services/provider-manager.service.ts | 42 +++++++++ src/processors/shared/rate-limiter.ts | 9 +- src/types/schemas.ts | 41 ++++++++- src/types/types.ts | 19 ++++ tests/integration/api/api-keys.test.ts | 34 +++++++ .../api/middlewares/auth_middleware.test.ts | 49 ++++++++++ tests/unit/api/jwt.utils.test.ts | 24 +++++ tests/unit/plugins/plugin-fs.test.ts | 92 +++++++++++++++++++ .../plugins/provider-manager.service.test.ts | 58 ++++++++++++ tests/unit/plugins/registry.test.ts | 22 +++++ 25 files changed, 676 insertions(+), 63 deletions(-) create mode 100644 tests/unit/plugins/plugin-fs.test.ts diff --git a/.env.example b/.env.example index c673ec7..9a4a15d 100644 --- a/.env.example +++ b/.env.example @@ -14,7 +14,7 @@ # ============================================ NODE_ENV=development # Set to 'production', 'development', or 'test' PORT=3000 # API server port -JWT_SECRET= # Secret key for JWT admin tokens (e.g. openssl rand -base64 32) +JWT_SECRET= # REQUIRED: Secret key for JWT admin tokens (min 32 chars, generate with: openssl rand -base64 32) # ============================================ # DATABASE diff --git a/dashboard/app/keys/page.tsx b/dashboard/app/keys/page.tsx index e14f426..5d0ea54 100644 --- a/dashboard/app/keys/page.tsx +++ b/dashboard/app/keys/page.tsx @@ -54,7 +54,7 @@ import { } from "lucide-react"; import { toast } from "sonner"; import { apiClient, apiKeyService } from "@/lib/api-client"; -import type { ApiKey, ApiKeyUsageDetailResponse } from "@/lib/types"; +import type { ApiKey, ApiKeyUsageDetailResponse, ApiKeyType } from "@/lib/types"; const fetcher = (url: string): Promise => apiClient.get(url) as unknown as Promise; @@ -65,6 +65,7 @@ export default function ApiKeysPage() { // Create Modal state const [createOpen, setCreateOpen] = useState(false); const [newKeyName, setNewKeyName] = useState(""); + const [newKeyType, setNewKeyType] = useState("notification"); const [isCreating, setIsCreating] = useState(false); // Reveal Key Modal state @@ -87,10 +88,11 @@ export default function ApiKeysPage() { setIsCreating(true); try { - const res = await apiKeyService.create({ name: newKeyName.trim() }); + const res = await apiKeyService.create({ name: newKeyName.trim(), type: newKeyType }); toast.success("API key created successfully"); setCreateOpen(false); setNewKeyName(""); + setNewKeyType("notification"); setRevealedKey({ name: res.key.name, raw_key: res.raw_key }); mutate(); } catch (err: unknown) { @@ -226,6 +228,7 @@ export default function ApiKeysPage() { Name Key Prefix + Type Status Usage (Notifications) Channel Breakdown @@ -248,6 +251,17 @@ export default function ApiKeysPage() { {key.key_prefix} + + {key.type === "admin" ? ( + + Admin (Full Access) + + ) : ( + + Notification + + )} + {key.status === "active" ? ( @@ -336,15 +350,55 @@ export default function ApiKeysPage() { Enter a descriptive name for this API key. The key secret will be displayed once after creation. -
- - setNewKeyName(e.target.value)} - required - autoFocus - /> +
+
+ + setNewKeyName(e.target.value)} + required + autoFocus + /> +
+ +
+ +
+
setNewKeyType("notification")} + className={`cursor-pointer rounded-lg border p-3 transition-colors ${ + newKeyType === "notification" + ? "border-primary bg-primary/5 text-primary" + : "border-border hover:bg-muted/50" + }`} + > +
+ Notification + {newKeyType === "notification" && } +
+

+ Can send notifications and check delivery status. +

+
+
setNewKeyType("admin")} + className={`cursor-pointer rounded-lg border p-3 transition-colors ${ + newKeyType === "admin" + ? "border-primary bg-primary/5 text-primary" + : "border-border hover:bg-muted/50" + }`} + > +
+ Admin (Full Access) + {newKeyType === "admin" && } +
+

+ Full access to management endpoints and notifications. +

+
+
+
+ +
+ + + updateGroupField( + "recovery", + "recovery_claim_timeout_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 60,000 (instance claim lock) +
+ +
+
+ + + updateGroupField( + "recovery", + "cleanup_resolved_alerts_retention_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 86,400,000 (24h) +
+ +
+ + + updateGroupField( + "recovery", + "cleanup_processed_status_outbox_retention_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 86,400,000 (24h) +
+
diff --git a/dashboard/lib/types.ts b/dashboard/lib/types.ts index 0d435c3..3cc5ed5 100644 --- a/dashboard/lib/types.ts +++ b/dashboard/lib/types.ts @@ -412,9 +412,12 @@ export interface ApiKeyUsage { last_used_at?: string | null; } +export type ApiKeyType = 'notification' | 'admin'; + export interface ApiKey { key_id: string; name: string; + type: ApiKeyType; key_prefix: string; status: 'active' | 'revoked'; expires_at?: string | null; @@ -424,6 +427,7 @@ export interface ApiKey { export interface CreateApiKeyPayload { name: string; + type?: ApiKeyType; expires_at?: string; } diff --git a/docker-compose/docker-compose.dev.yaml b/docker-compose/docker-compose.dev.yaml index 4402c0d..2a09903 100644 --- a/docker-compose/docker-compose.dev.yaml +++ b/docker-compose/docker-compose.dev.yaml @@ -24,7 +24,7 @@ services: BROKERS: kafka:9093 LOKI_URL: http://loki:3100 LOG_LEVEL: ${LOG_LEVEL:-info} - JWT_SECRET: ${JWT_SECRET:-simplens-default-jwt-secret-key-change-in-production} + JWT_SECRET: ${JWT_SECRET} SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} volumes: - logs-data-dev:/app/logs diff --git a/docker-compose/docker-compose.local.yaml b/docker-compose/docker-compose.local.yaml index 9d000ab..e0e8398 100644 --- a/docker-compose/docker-compose.local.yaml +++ b/docker-compose/docker-compose.local.yaml @@ -27,7 +27,7 @@ services: BROKERS: kafka:9093 LOKI_URL: http://loki:3100 LOG_LEVEL: ${LOG_LEVEL:-info} - JWT_SECRET: ${JWT_SECRET:-simplens-default-jwt-secret-key-change-in-production} + JWT_SECRET: ${JWT_SECRET} volumes: - logs-data-local:/app/logs command: [ "node", "dist/api/server.js" ] diff --git a/docker-compose/docker-compose.master.yaml b/docker-compose/docker-compose.master.yaml index c7173e0..c09b457 100644 --- a/docker-compose/docker-compose.master.yaml +++ b/docker-compose/docker-compose.master.yaml @@ -24,7 +24,7 @@ services: BROKERS: kafka:9093 LOKI_URL: http://loki:3100 LOG_LEVEL: ${LOG_LEVEL:-info} - JWT_SECRET: ${JWT_SECRET:-simplens-default-jwt-secret-key-change-in-production} + JWT_SECRET: ${JWT_SECRET} SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} volumes: - logs-data-master:/app/logs diff --git a/src/api/controllers/api-key.controller.ts b/src/api/controllers/api-key.controller.ts index 18f320e..b65ba10 100644 --- a/src/api/controllers/api-key.controller.ts +++ b/src/api/controllers/api-key.controller.ts @@ -20,6 +20,7 @@ function toApiKeyResponse(doc: ApiKeyDoc): ApiKeyResponse { return { key_id: doc.key_id, name: doc.name, + type: doc.type || 'notification', key_prefix: doc.key_prefix, status: doc.status, expires_at: doc.expires_at || null, @@ -63,7 +64,7 @@ export const createApiKey = async (req: Request, res: Response): Promise = return; } - const { name, expires_at } = validation.data; + const { name, type, expires_at } = validation.data; // Generate cryptographically random API key const rawKey = `sns_live_${crypto.randomBytes(24).toString('base64url')}`; @@ -74,6 +75,7 @@ export const createApiKey = async (req: Request, res: Response): Promise = const newDoc = await api_key_model.create({ key_id, name, + type: type || 'notification', key_prefix, key_hash, status: 'active', diff --git a/src/api/controllers/plugins.controller.ts b/src/api/controllers/plugins.controller.ts index ce6d58c..a544a10 100644 --- a/src/api/controllers/plugins.controller.ts +++ b/src/api/controllers/plugins.controller.ts @@ -8,7 +8,12 @@ import type { Request, Response } from 'express'; import { PluginRegistry, PluginManagerService, NpmAuthService } from '@src/plugins/index.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; import axios from 'axios'; -import { installPluginPayloadSchema, npmAuthConfigSchema } from '@src/types/schemas.js'; +import { + installPluginPayloadSchema, + npmAuthConfigSchema, + changePluginVersionSchema, + uninstallPluginSchema, +} from '@src/types/schemas.js'; import type { plugin_catalog_entry } from '@src/types/types.js'; const PLUGIN_CATALOG_BASE_URL = 'https://www.simplens.in/plugins'; @@ -132,15 +137,18 @@ export const installPlugin = async (req: Request, res: Response): Promise */ export const changePluginVersion = async (req: Request, res: Response): Promise => { try { - const { package: packageName, version } = req.body; - if (!packageName || !version) { + const validation = changePluginVersionSchema.safeParse(req.body); + if (!validation.success) { res.status(400).json({ error: 'Bad Request', - message: 'Both package name and target version are required' + message: 'Invalid plugin version change payload', + details: validation.error.flatten().fieldErrors, }); return; } + const { package: packageName, version } = validation.data; + const plugin = await PluginManagerService.changePluginVersion(packageName.trim(), version.trim()); res.json({ message: `Plugin '${packageName}' changed to version '${version}'`, @@ -162,24 +170,28 @@ export const changePluginVersion = async (req: Request, res: Response): Promise< */ export const uninstallPlugin = async (req: Request, res: Response): Promise => { try { - let packageName = (req.query.package as string) || (req.body?.package as string); + let rawPackageName = (req.query.package as string) || (req.body?.package as string); - if (!packageName) { + if (!rawPackageName) { if (req.params.scope && req.params.package) { - packageName = `${req.params.scope}/${req.params.package}`; + rawPackageName = `${req.params.scope}/${req.params.package}`; } else if (req.params.package) { - packageName = decodeURIComponent(req.params.package); + rawPackageName = decodeURIComponent(req.params.package); } } - if (!packageName || typeof packageName !== 'string') { + const validation = uninstallPluginSchema.safeParse({ package: rawPackageName }); + if (!validation.success) { res.status(400).json({ error: 'Bad Request', - message: 'A valid npm package name is required' + message: 'Invalid npm package name format', + details: validation.error.flatten().fieldErrors, }); return; } + const { package: packageName } = validation.data; + await PluginManagerService.uninstallPlugin(packageName.trim()); res.json({ message: `Plugin '${packageName.trim()}' uninstalled successfully` diff --git a/src/api/middlewares/auth_middleware.ts b/src/api/middlewares/auth_middleware.ts index cf325d8..b9fce00 100644 --- a/src/api/middlewares/auth_middleware.ts +++ b/src/api/middlewares/auth_middleware.ts @@ -43,14 +43,33 @@ async function handleDynamicApiKey( return; } - const path = req.originalUrl || req.path; - if (path.includes('/api/admin/auth/setup') || path.includes('/api/admin/auth/verify')) { + const fullPath = (req.originalUrl || `${req.baseUrl || ''}${req.path}`).split('?')[0]; + if (fullPath.includes('/api/admin/auth/setup') || fullPath.includes('/api/admin/auth/verify')) { res.status(403).json({ error: 'Forbidden: API keys cannot access administrator credential setup or verification routes.', }); return; } + const keyType = keyDoc.type || 'notification'; + const isManagementRoute = + fullPath.includes('/api/providers') || + fullPath.includes('/api/plugins') || + fullPath.includes('/api/channels/routing') || + fullPath.includes('/api/admin-channels') || + fullPath.includes('/api/settings') || + fullPath.includes('/api/keys') || + fullPath.includes('/api/dashboard') || + fullPath.includes('/api/alerts') || + (fullPath.includes('/api/templates') && req.method !== 'GET'); + + if (isManagementRoute && keyType !== 'admin') { + res.status(403).json({ + error: 'Forbidden: API key does not have management permissions. Requires admin key.', + }); + return; + } + req.apiKey = keyDoc.toObject() as ApiKeyDoc; void api_key_model.updateOne( diff --git a/src/api/server.ts b/src/api/server.ts index a37276b..273e245 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -87,6 +87,9 @@ app.use('/api/admin/auth', admin_auth_router); const start_server = async () => { try { + // Fail startup immediately if JWT_SECRET is missing or insecure + void env.JWT_SECRET; + const db = await connectMongoDB(); logger.success("Successfully connected to MongoDB"); diff --git a/src/config/env.config.ts b/src/config/env.config.ts index 9293b27..9f3c07c 100644 --- a/src/config/env.config.ts +++ b/src/config/env.config.ts @@ -7,6 +7,36 @@ dotenv.config(); // Unique worker/processor ID for this instance (used for distributed locking) const WORKER_ID = process.env.WORKER_ID || `worker-${randomUUID().slice(0, 8)}`; +export const INSECURE_JWT_SECRET_FALLBACK = "simplens-default-jwt-secret-key-change-in-production"; + +/** + * Validates that JWT_SECRET is explicitly configured and cryptographically strong. + * Throws a fatal error if missing, set to insecure default, or too short. + */ +export function validateJwtSecret(customSecret?: string): string { + const secret = (customSecret !== undefined ? customSecret : process.env.JWT_SECRET)?.trim(); + + if (!secret) { + throw new Error( + "FATAL: JWT_SECRET environment variable is missing. SimpleNS requires an explicitly configured JWT secret to secure admin tokens." + ); + } + + if (secret === INSECURE_JWT_SECRET_FALLBACK) { + throw new Error( + "FATAL: JWT_SECRET is set to the default insecure placeholder. You must configure a unique, secure secret." + ); + } + + if (secret.length < 32) { + throw new Error( + `FATAL: JWT_SECRET is too short (${secret.length} characters). It must be at least 32 characters long for cryptographic security.` + ); + } + + return secret; +} + export const env = { // ======================================================================== // 1. BOOTSTRAP ENVIRONMENT VARIABLES (from .env / process.env) @@ -23,7 +53,9 @@ export const env = { // API Server PORT: parseInt(process.env.PORT || "3000"), - JWT_SECRET: (process.env.JWT_SECRET || "simplens-default-jwt-secret-key-change-in-production"), + get JWT_SECRET(): string { + return validateJwtSecret(); + }, DELAYED_PARTITION: parseInt(process.env.DELAYED_PARTITION || "1"), NOTIFICATION_STATUS_PARTITION: parseInt(process.env.NOTIFICATION_STATUS_PARTITION || "1"), diff --git a/src/database/models/api-key.models.ts b/src/database/models/api-key.models.ts index f0dc1c7..615e7ad 100644 --- a/src/database/models/api-key.models.ts +++ b/src/database/models/api-key.models.ts @@ -19,6 +19,12 @@ const api_key_schema = new mongoose.Schema( type: String, required: true, }, + type: { + type: String, + enum: ['notification', 'admin'], + default: 'notification', + index: true, + }, key_prefix: { type: String, required: true, diff --git a/src/plugins/loader/plugin-fs.ts b/src/plugins/loader/plugin-fs.ts index c84d139..3f5a9e8 100644 --- a/src/plugins/loader/plugin-fs.ts +++ b/src/plugins/loader/plugin-fs.ts @@ -7,10 +7,14 @@ import { readFileSync, existsSync, mkdirSync, writeFileSync } from 'fs'; import { join } from 'path'; -import { execSync } from 'child_process'; +import { spawnSync } from 'child_process'; import { pathToFileURL } from 'url'; import type { SimpleNSProvider, ProviderManifest } from '@src/types/types.js'; -import { providerManifestSchema } from '@src/types/schemas.js'; +import { + providerManifestSchema, + npmPackageNameSchema, + npmPackageVersionSchema, +} from '@src/types/schemas.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; export const PLUGINS_DIR = join(process.cwd(), '.plugins'); @@ -279,20 +283,31 @@ export async function extractPackageManifest( * Install an npm package into .plugins */ export function installNpmPackage(packageName: string, version?: string): void { + const pkgValidation = npmPackageNameSchema.safeParse(packageName); + if (!pkgValidation.success) { + throw new Error(`Invalid npm package name '${packageName}'`); + } + if (version && version !== 'latest') { + const verValidation = npmPackageVersionSchema.safeParse(version); + if (!verValidation.success) { + throw new Error(`Invalid npm package version '${version}'`); + } + } + initPluginsDir(); const specifier = version && version !== 'latest' ? `${packageName}@${version}` : `${packageName}@latest`; logger.info(`Running npm install ${specifier} in ${PLUGINS_DIR}...`); - try { - execSync(`npm install ${specifier}`, { - cwd: PLUGINS_DIR, - stdio: 'pipe', - }); - logger.success(`Successfully installed ${specifier}`); - } catch (err: unknown) { - const execErr = err as { stderr?: Buffer; message?: string }; - const stderrMsg = execErr.stderr ? execErr.stderr.toString('utf-8') : ''; - logger.error(`Failed to install ${specifier}:`, stderrMsg || execErr.message || err); + const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm'; + const result = spawnSync(npmCmd, ['install', specifier], { + cwd: PLUGINS_DIR, + stdio: 'pipe', + shell: process.platform === 'win32', + }); + + if (result.status !== 0 || result.error) { + const stderrMsg = result.stderr ? result.stderr.toString('utf-8') : (result.error?.message || ''); + logger.error(`Failed to install ${specifier}:`, stderrMsg || result.error); let friendlyMessage = `Failed to install package '${specifier}'`; if (stderrMsg.includes('E401') || stderrMsg.includes('401 Unauthorized')) { @@ -305,25 +320,36 @@ export function installNpmPackage(packageName: string, version?: string): void { friendlyMessage += `: ${stderrMsg.trim().slice(0, 300)}`; } - throw new Error(friendlyMessage, { cause: err }); + throw new Error(friendlyMessage, { cause: result.error }); } + + logger.success(`Successfully installed ${specifier}`); } /** * Uninstall an npm package from .plugins */ export function uninstallNpmPackage(packageName: string): void { + const pkgValidation = npmPackageNameSchema.safeParse(packageName); + if (!pkgValidation.success) { + throw new Error(`Invalid npm package name '${packageName}'`); + } + initPluginsDir(); logger.info(`Running npm uninstall ${packageName} in ${PLUGINS_DIR}...`); - try { - execSync(`npm uninstall ${packageName}`, { - cwd: PLUGINS_DIR, - stdio: 'pipe', - }); - logger.success(`Successfully uninstalled ${packageName}`); - } catch (err) { - logger.error(`Failed to uninstall ${packageName}:`, err); - throw new Error(`Failed to uninstall package: ${packageName}`, { cause: err }); + const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm'; + const result = spawnSync(npmCmd, ['uninstall', packageName], { + cwd: PLUGINS_DIR, + stdio: 'pipe', + shell: process.platform === 'win32', + }); + + if (result.status !== 0 || result.error) { + const stderrMsg = result.stderr ? result.stderr.toString('utf-8') : (result.error?.message || ''); + logger.error(`Failed to uninstall ${packageName}:`, stderrMsg || result.error); + throw new Error(`Failed to uninstall package: ${packageName}`, { cause: result.error }); } + + logger.success(`Successfully uninstalled ${packageName}`); } diff --git a/src/plugins/loader/registry.ts b/src/plugins/loader/registry.ts index 245eeef..7516584 100644 --- a/src/plugins/loader/registry.ts +++ b/src/plugins/loader/registry.ts @@ -201,8 +201,23 @@ class PluginRegistryClass { getDefaultProvider(channel: string): SimpleNSProvider | undefined { const config = this.channelConfig.get(channel); if (config?.default) { - return this.get(config.default); + const defaultProvider = this.get(config.default); + if (defaultProvider) { + return defaultProvider; + } + } + + // If configured default is unregistered/disabled, check explicitly configured fallbacks in order + if (config?.fallback) { + const fallbackIds = Array.isArray(config.fallback) ? config.fallback : [config.fallback]; + for (const fallbackId of fallbackIds) { + const fallbackProvider = this.get(fallbackId); + if (fallbackProvider) { + return fallbackProvider; + } + } } + // Fall back to first registered provider for channel const providers = this.getProvidersForChannel(channel); return providers[0]; @@ -213,9 +228,20 @@ class PluginRegistryClass { */ getDefaultProviderId(channel: string): string | undefined { const config = this.channelConfig.get(channel); - if (config?.default) { + if (config?.default && this.get(config.default)) { return config.default; } + + // If configured default is unregistered/disabled, check explicitly configured fallbacks in order + if (config?.fallback) { + const fallbackIds = Array.isArray(config.fallback) ? config.fallback : [config.fallback]; + for (const fallbackId of fallbackIds) { + if (this.get(fallbackId)) { + return fallbackId; + } + } + } + // Fall back to first registered provider ID for channel const ids = this.channelProviders.get(channel); return ids?.[0]; diff --git a/src/plugins/services/provider-manager.service.ts b/src/plugins/services/provider-manager.service.ts index d40da92..2640603 100644 --- a/src/plugins/services/provider-manager.service.ts +++ b/src/plugins/services/provider-manager.service.ts @@ -183,6 +183,48 @@ export class ProviderManagerService { } } else { PluginRegistry.unregister(id); + + // If disabling a provider that was currently default in ChannelRouting, promote an active fallback + try { + const routings = await ChannelRouting.find({ default_provider_id: id }).lean(); + for (const routing of routings) { + const enabledProviders = await Provider.find({ + channel: routing.channel, + enabled: true, + id: { $ne: id }, + }).lean(); + const enabledProviderIds = new Set(enabledProviders.map((p) => p.id)); + + const remainingFallbacks = (routing.fallback_provider_ids || []).filter( + (pid: string) => pid !== id + ); + const nextDefault = remainingFallbacks.find((pid: string) => enabledProviderIds.has(pid)) + || enabledProviders[0]?.id; + + if (nextDefault) { + const newFallbacks = remainingFallbacks.filter((pid: string) => pid !== nextDefault); + await ChannelRouting.updateOne( + { _id: routing._id }, + { + $set: { + default_provider_id: nextDefault, + fallback_provider_ids: newFallbacks, + }, + } + ); + PluginRegistry.setChannelConfig(routing.channel, { + default: nextDefault, + fallback: newFallbacks, + }); + await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { channel: routing.channel }); + logger.info(`Promoted fallback provider '${nextDefault}' as default for channel '${routing.channel}' upon disabling '${id}'.`); + } + } + } catch (routingErr) { + logger.warn(`Failed to promote fallback routing when disabling provider '${id}':`, { + error: routingErr instanceof Error ? routingErr.message : String(routingErr), + }); + } } await PluginSyncService.publish('PROVIDER_UPSERTED', { diff --git a/src/processors/shared/rate-limiter.ts b/src/processors/shared/rate-limiter.ts index d6a0106..3c4b3aa 100644 --- a/src/processors/shared/rate-limiter.ts +++ b/src/processors/shared/rate-limiter.ts @@ -17,6 +17,7 @@ import type { RateLimitResult, ProviderRateLimitStatus, ProviderRateLimitListResponse, + ProviderBasicMeta, } from '@src/types/types.js'; // Conversion divisors to convert rate to per-second @@ -187,14 +188,6 @@ export const getTokenCount = async (providerId: string): Promise => { return Math.min(currentTokens + tokensToAdd, config.maxTokens); }; -interface ProviderBasicMeta { - id: string; - channel: string; - plugin_name: string; - display_name?: string; - enabled: boolean; -} - /** * Fetch all provider rate limit statuses at scale using batched Redis MGET */ diff --git a/src/types/schemas.ts b/src/types/schemas.ts index 73d16f6..5bb24c2 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -615,9 +615,41 @@ export const npmAuthStatusSchema = z.object({ /** * Payload schema for installing a plugin package */ +/** + * Strict npm package name schema preventing shell injection and metacharacters + */ +export const npmPackageNameSchema = z.string() + .min(1, 'Package name is required') + .max(214, 'Package name cannot exceed 214 characters') + .regex(/^(?:@[a-z0-9_.-]+\/)?[a-z0-9_.-]+$/i, 'Invalid npm package name format'); + +/** + * Strict npm package version / tag schema + */ +export const npmPackageVersionSchema = z.string() + .regex( + /^(latest|\^?[0-9]+(?:\.[0-9]+)?(?:\.[0-9]+)?(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?|~?[0-9]+(?:\.[0-9]+)?(?:\.[0-9]+)?(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?|[0-9]+(?:\.[0-9]+)?(?:\.[0-9]+)?(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)$/, + 'Invalid npm version or tag format' + ); + +/** + * Payload schema for changing plugin version + */ +export const changePluginVersionSchema = z.object({ + package: npmPackageNameSchema, + version: npmPackageVersionSchema, +}); + +/** + * Schema for uninstalling a plugin + */ +export const uninstallPluginSchema = z.object({ + package: npmPackageNameSchema, +}); + export const installPluginPayloadSchema = z.object({ - package: z.string().min(1, "Package name is required"), - version: z.string().optional(), + package: npmPackageNameSchema, + version: npmPackageVersionSchema.optional(), auth: z.object({ token: z.string().optional(), registry_url: z.string().url("Invalid registry URL").optional(), @@ -694,9 +726,12 @@ export const apiKeyUsageSchema = z.object({ export const apiKeyStatusSchema = z.enum(['active', 'revoked']); +export const apiKeyTypeSchema = z.enum(['notification', 'admin']); + export const apiKeyDocSchema = z.object({ key_id: z.string(), name: z.string(), + type: apiKeyTypeSchema.default('notification'), key_prefix: z.string(), key_hash: z.string(), status: apiKeyStatusSchema, @@ -708,12 +743,14 @@ export const apiKeyDocSchema = z.object({ export const createApiKeySchema = z.object({ name: z.string().min(1, 'API key name is required').max(100), + type: apiKeyTypeSchema.default('notification'), expires_at: z.coerce.date().optional(), }); export const apiKeyResponseSchema = z.object({ key_id: z.string(), name: z.string(), + type: apiKeyTypeSchema, key_prefix: z.string(), status: apiKeyStatusSchema, expires_at: z.coerce.date().nullable().optional(), diff --git a/src/types/types.ts b/src/types/types.ts index e6771ed..9c688bb 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -140,6 +140,10 @@ import { npmAuthConfigSchema, npmAuthStatusSchema, installPluginPayloadSchema, + changePluginVersionSchema, + uninstallPluginSchema, + npmPackageNameSchema, + npmPackageVersionSchema, providerRateLimitConfigDetailsSchema, providerRateLimitRealtimeStatusSchema, providerRateLimitStatusSchema, @@ -147,6 +151,7 @@ import { providerRateLimitListResponseSchema, apiKeyUsageSchema, apiKeyStatusSchema, + apiKeyTypeSchema, apiKeyDocSchema, createApiKeySchema, apiKeyResponseSchema, @@ -578,6 +583,19 @@ export type ProviderRateLimitStatus = z.infer; export type ProviderRateLimitListResponse = z.infer; +export interface ProviderBasicMeta { + id: string; + channel: string; + plugin_name: string; + display_name?: string; + enabled: boolean; +} + +export type npm_package_name = z.infer; +export type npm_package_version = z.infer; +export type change_plugin_version_payload = z.infer; +export type uninstall_plugin_payload = z.infer; + // ============================================================================ // API KEY & ADMIN AUTH TYPES // ============================================================================ @@ -590,6 +608,7 @@ export interface AdminJwtPayload { exp?: number; } +export type ApiKeyType = z.infer; export type ApiKeyStatus = z.infer; export type ApiKeyUsage = z.infer; export type ApiKeyDoc = z.infer; diff --git a/tests/integration/api/api-keys.test.ts b/tests/integration/api/api-keys.test.ts index 010963b..77cc81e 100644 --- a/tests/integration/api/api-keys.test.ts +++ b/tests/integration/api/api-keys.test.ts @@ -354,5 +354,39 @@ describe('API Keys Integration & Usage Tracking', () => { expect(statusRes.status).toBe(200); expect(statusRes.body.isConfigured).toBe(true); }); + + it('should forbid notification API keys from accessing /api/keys management route', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Worker Key', type: 'notification' }); + + const rawKey = createRes.body.raw_key; + expect(createRes.body.key.type).toBe('notification'); + + const keysRes = await request(app) + .get('/api/keys') + .set('Authorization', `Bearer ${rawKey}`); + + expect(keysRes.status).toBe(403); + expect(keysRes.body.error).toContain('management permissions'); + }); + + it('should allow admin API keys to access /api/keys management route', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Super Admin Key', type: 'admin' }); + + const rawKey = createRes.body.raw_key; + expect(createRes.body.key.type).toBe('admin'); + + const keysRes = await request(app) + .get('/api/keys') + .set('Authorization', `Bearer ${rawKey}`); + + expect(keysRes.status).toBe(200); + expect(keysRes.body.keys).toBeDefined(); + }); }); }); diff --git a/tests/integration/api/middlewares/auth_middleware.test.ts b/tests/integration/api/middlewares/auth_middleware.test.ts index 44dccd1..8fc7ee4 100644 --- a/tests/integration/api/middlewares/auth_middleware.test.ts +++ b/tests/integration/api/middlewares/auth_middleware.test.ts @@ -166,6 +166,55 @@ describe('Auth Middleware', () => { expect(res.status).toHaveBeenCalledWith(403); expect(next).not.toHaveBeenCalled(); }); + + it('should return 403 when notification API key accesses management routes', async () => { + const rawKey = 'sns_live_notifkey'; + const mockKeyDoc = { + key_id: 'key-123', + status: 'active', + type: 'notification', + expires_at: null, + toObject: () => ({ key_id: 'key-123', type: 'notification' }), + }; + + mockFindOne.mockResolvedValueOnce(mockKeyDoc); + + const req = createMockRequest(`Bearer ${rawKey}`, '/api/providers') as Request; + const res = createMockResponse() as Response; + const next = vi.fn(); + + await authMiddleware.auth_middleware(req, res, next as NextFunction); + + expect(res.status).toHaveBeenCalledWith(403); + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ + error: expect.stringContaining('Requires admin key'), + }) + ); + expect(next).not.toHaveBeenCalled(); + }); + + it('should allow admin API key to access management routes', async () => { + const rawKey = 'sns_live_adminkey'; + const mockKeyDoc = { + key_id: 'key-admin', + status: 'active', + type: 'admin', + expires_at: null, + toObject: () => ({ key_id: 'key-admin', type: 'admin' }), + }; + + mockFindOne.mockResolvedValueOnce(mockKeyDoc); + + const req = createMockRequest(`Bearer ${rawKey}`, '/api/providers') as Request; + const res = createMockResponse() as Response; + const next = vi.fn(); + + await authMiddleware.auth_middleware(req, res, next as NextFunction); + + expect(next).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + }); }); describe('Missing or Invalid Authentication', () => { diff --git a/tests/unit/api/jwt.utils.test.ts b/tests/unit/api/jwt.utils.test.ts index f66c7e9..76ae252 100644 --- a/tests/unit/api/jwt.utils.test.ts +++ b/tests/unit/api/jwt.utils.test.ts @@ -40,4 +40,28 @@ describe('Admin JWT Utilities', () => { expect(verifyAdminJwt('')).toBeNull(); expect(verifyAdminJwt('random-string')).toBeNull(); }); + + describe('JWT Secret Validation', () => { + it('should fail if JWT_SECRET is empty or whitespace', async () => { + const { validateJwtSecret } = await import('@src/config/env.config.js'); + expect(() => validateJwtSecret('')).toThrow(/missing/i); + expect(() => validateJwtSecret(' ')).toThrow(/missing/i); + }); + + it('should fail if JWT_SECRET is the default insecure placeholder', async () => { + const { validateJwtSecret, INSECURE_JWT_SECRET_FALLBACK } = await import('@src/config/env.config.js'); + expect(() => validateJwtSecret(INSECURE_JWT_SECRET_FALLBACK)).toThrow(/insecure placeholder/i); + }); + + it('should fail if JWT_SECRET is shorter than 32 characters', async () => { + const { validateJwtSecret } = await import('@src/config/env.config.js'); + expect(() => validateJwtSecret('short-secret-key-123')).toThrow(/too short/i); + }); + + it('should succeed when JWT_SECRET is at least 32 characters', async () => { + const { validateJwtSecret } = await import('@src/config/env.config.js'); + const valid = 'a-very-strong-and-secure-random-jwt-secret-key-32-chars'; + expect(validateJwtSecret(valid)).toBe(valid); + }); + }); }); diff --git a/tests/unit/plugins/plugin-fs.test.ts b/tests/unit/plugins/plugin-fs.test.ts new file mode 100644 index 0000000..a5da289 --- /dev/null +++ b/tests/unit/plugins/plugin-fs.test.ts @@ -0,0 +1,92 @@ +/** + * Unit tests for plugin-fs.ts + * Tests package name and version validation and spawnSync command execution + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockSpawnSync = vi.fn(); +vi.mock('child_process', () => ({ + spawnSync: (...args: unknown[]) => mockSpawnSync(...args), + readFileSync: vi.fn(), + existsSync: vi.fn().mockReturnValue(true), + mkdirSync: vi.fn(), + writeFileSync: vi.fn(), +})); + +describe('Plugin FS Package Utilities', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockSpawnSync.mockReturnValue({ + status: 0, + stderr: Buffer.from(''), + stdout: Buffer.from(''), + }); + }); + + describe('installNpmPackage', () => { + it('should reject package names with shell injection characters', async () => { + const { installNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + expect(() => installNpmPackage('express; rm -rf /')).toThrow(/invalid npm package name/i); + expect(() => installNpmPackage('express && calc')).toThrow(/invalid npm package name/i); + expect(() => installNpmPackage('express | whoami')).toThrow(/invalid npm package name/i); + expect(() => installNpmPackage('`calc`')).toThrow(/invalid npm package name/i); + expect(() => installNpmPackage('$(whoami)')).toThrow(/invalid npm package name/i); + expect(() => installNpmPackage('foo/bar/baz')).toThrow(/invalid npm package name/i); + expect(mockSpawnSync).not.toHaveBeenCalled(); + }); + + it('should reject version strings with shell injection characters', async () => { + const { installNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + expect(() => installNpmPackage('lodash', '1.0.0; calc')).toThrow(/invalid npm package version/i); + expect(() => installNpmPackage('lodash', '^1.0.0 & whoami')).toThrow(/invalid npm package version/i); + expect(() => installNpmPackage('lodash', '$(cat /etc/passwd)')).toThrow(/invalid npm package version/i); + expect(mockSpawnSync).not.toHaveBeenCalled(); + }); + + it('should invoke spawnSync with argument array for valid packages', async () => { + const { installNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + installNpmPackage('lodash', '4.17.21'); + + expect(mockSpawnSync).toHaveBeenCalledTimes(1); + const [cmd, args] = mockSpawnSync.mock.calls[0]; + expect(cmd).toMatch(/npm(\.cmd)?$/); + expect(args).toEqual(['install', 'lodash@4.17.21']); + }); + + it('should invoke spawnSync with argument array for valid scoped packages', async () => { + const { installNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + installNpmPackage('@simplens/provider-smtp', '^1.2.0'); + + expect(mockSpawnSync).toHaveBeenCalledTimes(1); + const [cmd, args] = mockSpawnSync.mock.calls[0]; + expect(cmd).toMatch(/npm(\.cmd)?$/); + expect(args).toEqual(['install', '@simplens/provider-smtp@^1.2.0']); + }); + }); + + describe('uninstallNpmPackage', () => { + it('should reject package names with shell injection characters', async () => { + const { uninstallNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + expect(() => uninstallNpmPackage('lodash; rm -rf /')).toThrow(/invalid npm package name/i); + expect(() => uninstallNpmPackage('foo & calc')).toThrow(/invalid npm package name/i); + expect(mockSpawnSync).not.toHaveBeenCalled(); + }); + + it('should invoke spawnSync with argument array for valid package', async () => { + const { uninstallNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + uninstallNpmPackage('@simplens/provider-smtp'); + + expect(mockSpawnSync).toHaveBeenCalledTimes(1); + const [cmd, args] = mockSpawnSync.mock.calls[0]; + expect(cmd).toMatch(/npm(\.cmd)?$/); + expect(args).toEqual(['uninstall', '@simplens/provider-smtp']); + }); + }); +}); diff --git a/tests/unit/plugins/provider-manager.service.test.ts b/tests/unit/plugins/provider-manager.service.test.ts index 44e3d03..ef8c56f 100644 --- a/tests/unit/plugins/provider-manager.service.test.ts +++ b/tests/unit/plugins/provider-manager.service.test.ts @@ -145,6 +145,64 @@ describe('ProviderManagerService', () => { }); }); + describe('updateProvider', () => { + it('should promote an active fallback provider to default in routing when default provider is disabled', async () => { + const mockDoc = { + _id: '507f1f77bcf86cd799439011', + id: 'gmail-prod', + plugin_name: '@simplens/mock', + channel: 'email', + enabled: true, + save: vi.fn().mockResolvedValue(undefined), + toObject: () => mockDoc, + }; + + vi.spyOn(Provider, 'findOne').mockResolvedValue(mockDoc as any); + vi.spyOn(ChannelRouting, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + _id: 'routing-1', + channel: 'email', + default_provider_id: 'gmail-prod', + fallback_provider_ids: ['smtp-backup', 'secondary-backup'], + }, + ]), + } as any); + + vi.spyOn(Provider, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { id: 'smtp-backup', channel: 'email', enabled: true }, + ]), + } as any); + + const updateRoutingSpy = vi.spyOn(ChannelRouting, 'updateOne').mockResolvedValue({ acknowledged: true } as any); + const setChannelConfigSpy = vi.spyOn(PluginRegistry, 'setChannelConfig'); + vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); + + await ProviderManagerService.updateProvider('gmail-prod', { enabled: false }); + + expect(mockDoc.enabled).toBe(false); + expect(mockDoc.save).toHaveBeenCalled(); + expect(PluginRegistry.unregister).toHaveBeenCalledWith('gmail-prod'); + expect(updateRoutingSpy).toHaveBeenCalledWith( + { _id: 'routing-1' }, + { + $set: { + default_provider_id: 'smtp-backup', + fallback_provider_ids: ['secondary-backup'], + }, + } + ); + expect(setChannelConfigSpy).toHaveBeenCalledWith('email', { + default: 'smtp-backup', + fallback: ['secondary-backup'], + }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('CHANNEL_ROUTING_UPDATED', { + channel: 'email', + }); + }); + }); + describe('deleteProvider', () => { it('should remove the provider from routing and promote the first fallback', async () => { vi.spyOn(Provider, 'findOne').mockResolvedValue({ id: 'gmail-prod' } as any); diff --git a/tests/unit/plugins/registry.test.ts b/tests/unit/plugins/registry.test.ts index 8c3c3d6..766e8ea 100644 --- a/tests/unit/plugins/registry.test.ts +++ b/tests/unit/plugins/registry.test.ts @@ -154,6 +154,18 @@ describe('Plugin Registry', () => { const provider = PluginRegistry.getDefaultProvider('unknown'); expect(provider).toBeUndefined(); }); + + it('should fall back to registered fallback provider when configured default is unregistered', () => { + const fallback = createMockProvider({ name: 'fallback', channel: 'email' }); + PluginRegistry.register(fallback as any, 'fallback', 1); + + // Configure default to an unregistered provider, with fallback configured + PluginRegistry.setChannelConfig('email', { default: 'unregistered-default', fallback: ['fallback'] }); + + const defaultProvider = PluginRegistry.getDefaultProvider('email'); + expect(defaultProvider).toBeDefined(); + expect(defaultProvider?.manifest.name).toBe('fallback'); + }); }); describe('getDefaultProviderId', () => { @@ -167,6 +179,16 @@ describe('Plugin Registry', () => { expect(defaultId).toBe('gmail'); }); + it('should return fallback provider ID when configured default is unregistered', () => { + const fallback = createMockProvider({ name: 'fallback', channel: 'email' }); + PluginRegistry.register(fallback as any, 'fallback', 1); + + PluginRegistry.setChannelConfig('email', { default: 'unregistered-default', fallback: ['fallback'] }); + + const defaultId = PluginRegistry.getDefaultProviderId('email'); + expect(defaultId).toBe('fallback'); + }); + it('should return undefined for unknown channel', () => { const defaultId = PluginRegistry.getDefaultProviderId('unknown'); expect(defaultId).toBeUndefined(); From be9de41c3d488fd6485bd330b7c93a10200df30d Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Fri, 2 Oct 2026 18:46:56 +0530 Subject: [PATCH 35/46] feat(api-key-cache): implement in-memory caching and usage aggregation for API keys --- scripts/load-test.js | 9 +- src/api/controllers/api-key.controller.ts | 3 + src/api/middlewares/auth_middleware.ts | 31 +-- src/api/server.ts | 4 + src/api/utils/api-key-cache.ts | 262 ++++++++++++++++++++++ src/api/utils/utils.ts | 91 +++++--- src/config/db.config.ts | 6 +- src/types/types.ts | 12 + 8 files changed, 362 insertions(+), 56 deletions(-) create mode 100644 src/api/utils/api-key-cache.ts diff --git a/scripts/load-test.js b/scripts/load-test.js index 3c3ec2f..fea9716 100644 --- a/scripts/load-test.js +++ b/scripts/load-test.js @@ -49,10 +49,6 @@ const projectRootDir = join(__dirname, '..'); dotenv.config({ path: join(projectRootDir, '.env') }); dotenv.config(); -// HTTP Connection Pooling for high concurrency load testing -const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 200 }); -const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 200 }); - // --- Argument Parsing --- const args = process.argv.slice(2); const config = { @@ -178,6 +174,10 @@ if (!webhookUrl) { webhookUrl = `http://${host}:4000/webhook`; } +// HTTP Connection Pooling for high concurrency load testing +const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 200 }); +const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 200 }); + // Axios Client instance with connection pooling const apiClient = axios.create({ baseURL: config.baseUrl, @@ -387,4 +387,3 @@ main().catch(err => { console.error('Fatal execution error:', err); process.exit(1); }); - diff --git a/src/api/controllers/api-key.controller.ts b/src/api/controllers/api-key.controller.ts index b65ba10..33360ad 100644 --- a/src/api/controllers/api-key.controller.ts +++ b/src/api/controllers/api-key.controller.ts @@ -11,6 +11,7 @@ import api_key_model from '@src/database/models/api-key.models.js'; import notification_model from '@src/database/models/notification.models.js'; import { createApiKeySchema } from '@src/types/schemas.js'; import type { ApiKeyDoc, ApiKeyResponse } from '@src/types/types.js'; +import { ApiKeyCache } from '../utils/api-key-cache.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; /** @@ -160,6 +161,7 @@ export const revokeApiKey = async (req: Request, res: Response): Promise = } logger.info(`Revoked API key: ${key.name} (${id})`); + ApiKeyCache.invalidate(id); res.status(200).json({ success: true, message: 'API key has been revoked successfully', @@ -185,6 +187,7 @@ export const deleteApiKey = async (req: Request, res: Response): Promise = } logger.info(`Deleted API key (${id})`); + ApiKeyCache.invalidate(id); res.status(200).json({ success: true, message: 'API key deleted successfully', diff --git a/src/api/middlewares/auth_middleware.ts b/src/api/middlewares/auth_middleware.ts index b9fce00..5141377 100644 --- a/src/api/middlewares/auth_middleware.ts +++ b/src/api/middlewares/auth_middleware.ts @@ -10,6 +10,7 @@ import type { Request, Response, NextFunction } from 'express'; import crypto from 'crypto'; import api_key_model from '@src/database/models/api-key.models.js'; import { verifyAdminJwt } from '../utils/jwt.utils.js'; +import { ApiKeyCache, ApiKeyUsageAggregator } from '../utils/api-key-cache.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; import type { ApiKeyDoc, AdminJwtPayload } from '@src/types/types.js'; @@ -31,14 +32,24 @@ async function handleDynamicApiKey( ): Promise { try { const tokenHash = crypto.createHash('sha256').update(token).digest('hex'); - const keyDoc = await api_key_model.findOne({ key_hash: tokenHash, status: 'active' }); - + + let keyDoc = ApiKeyCache.get(tokenHash); if (!keyDoc) { - res.status(401).json({ message: 'Invalid API KEY' }); - return; + const dbDoc = await api_key_model.findOne({ key_hash: tokenHash, status: 'active' }); + if (!dbDoc) { + res.status(401).json({ message: 'Invalid API KEY' }); + return; + } + const rawExpiresAt = (dbDoc as unknown as { expires_at?: Date | string | null }).expires_at; + keyDoc = (typeof dbDoc.toObject === 'function' ? dbDoc.toObject() : dbDoc) as ApiKeyDoc; + if (rawExpiresAt && !keyDoc.expires_at) { + keyDoc.expires_at = typeof rawExpiresAt === 'string' ? new Date(rawExpiresAt) : rawExpiresAt; + } + ApiKeyCache.set(tokenHash, keyDoc); } if (keyDoc.expires_at && new Date() > new Date(keyDoc.expires_at)) { + ApiKeyCache.invalidate(tokenHash); res.status(401).json({ message: 'API key has expired' }); return; } @@ -70,17 +81,9 @@ async function handleDynamicApiKey( return; } - req.apiKey = keyDoc.toObject() as ApiKeyDoc; + req.apiKey = keyDoc; - void api_key_model.updateOne( - { key_id: keyDoc.key_id }, - { - $set: { 'usage.last_used_at': new Date() }, - $inc: { 'usage.total_requests': 1 }, - } - ).catch((err: unknown) => { - logger.warn(`Failed to update usage for API key ${keyDoc.key_id}`, { error: String(err) }); - }); + ApiKeyUsageAggregator.recordRequest(keyDoc.key_id); next(); } catch (err) { diff --git a/src/api/server.ts b/src/api/server.ts index 273e245..4c3f722 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -27,6 +27,7 @@ import { disconnectRedis } from '@src/config/redis.config.js'; import { createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; import type { HealthProbeServer } from '@src/types/types.js'; import { auth_middleware } from './middlewares/auth_middleware.js'; +import { ApiKeyUsageAggregator } from './utils/api-key-cache.js'; import http from 'http'; import helmet from 'helmet'; import cors from 'cors'; @@ -117,6 +118,8 @@ const start_server = async () => { await createTopics(topics); const server = http.createServer(app); + server.keepAliveTimeout = 65000; + server.headersTimeout = 66000; server.listen(env.PORT, () => logger.success(`Notification Service running at http://localhost:${env.PORT}`)); // 5. Start health probe server for Kubernetes @@ -144,6 +147,7 @@ const start_server = async () => { }); }); } + await ApiKeyUsageAggregator.flush(); await disconnectRedis(); await db.disconnect(); } catch (e) { diff --git a/src/api/utils/api-key-cache.ts b/src/api/utils/api-key-cache.ts new file mode 100644 index 0000000..e24433c --- /dev/null +++ b/src/api/utils/api-key-cache.ts @@ -0,0 +1,262 @@ +/** + * In-Memory API Key Cache & Debounced Usage Aggregator + * + * Provides high-performance in-memory caching of validated API keys and + * debounced batch aggregation of usage metrics to prevent MongoDB document lock + * contention and connection pool exhaustion under heavy concurrency. + */ + +import api_key_model from '@src/database/models/api-key.models.js'; +import type { ApiKeyDoc, ApiKeyCacheEntry, PendingApiKeyUsage } from '@src/types/types.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +const DEFAULT_CACHE_TTL_MS = 60 * 1000; // 60 seconds +const DEFAULT_FLUSH_INTERVAL_MS = 1000; // 1 second +const DEFAULT_MAX_BATCH_THRESHOLD = 200; // flush early if pending requests exceed this threshold + +export class ApiKeyCache { + private static cacheByHash: Map = new Map(); + private static keyIdToHash: Map = new Map(); + private static ttlMs: number = DEFAULT_CACHE_TTL_MS; + + /** + * Set cache TTL in milliseconds (useful for testing) + */ + public static setTtlMs(ttl: number): void { + this.ttlMs = ttl; + } + + /** + * Retrieve active API key document from memory cache if not expired. + */ + public static get(tokenHash: string): ApiKeyDoc | null { + const entry = this.cacheByHash.get(tokenHash); + if (!entry) return null; + + if (Date.now() - entry.cachedAt > this.ttlMs) { + this.cacheByHash.delete(tokenHash); + this.keyIdToHash.delete(entry.keyDoc.key_id); + return null; + } + + return entry.keyDoc; + } + + /** + * Cache validated API key document in memory. + */ + public static set(tokenHash: string, keyDoc: ApiKeyDoc): void { + this.cacheByHash.set(tokenHash, { + keyDoc, + cachedAt: Date.now(), + }); + this.keyIdToHash.set(keyDoc.key_id, tokenHash); + } + + /** + * Invalidate cached API key by key_id or tokenHash. + */ + public static invalidate(keyIdOrHash: string): void { + if (this.cacheByHash.has(keyIdOrHash)) { + const entry = this.cacheByHash.get(keyIdOrHash); + if (entry) { + this.keyIdToHash.delete(entry.keyDoc.key_id); + } + this.cacheByHash.delete(keyIdOrHash); + return; + } + + const hash = this.keyIdToHash.get(keyIdOrHash); + if (hash) { + this.cacheByHash.delete(hash); + this.keyIdToHash.delete(keyIdOrHash); + } + } + + /** + * Clear all cached keys. + */ + public static clear(): void { + this.cacheByHash.clear(); + this.keyIdToHash.clear(); + } +} + +export class ApiKeyUsageAggregator { + private static pendingUsage: Map = new Map(); + private static flushTimer: NodeJS.Timeout | null = null; + private static isFlushing = false; + + private static getOrCreate(keyId: string): PendingApiKeyUsage { + let entry = this.pendingUsage.get(keyId); + if (!entry) { + entry = { + total_requests: 0, + total_notifications: 0, + by_channel: {}, + last_used_at: new Date(), + }; + this.pendingUsage.set(keyId, entry); + } + return entry; + } + + private static isTestEnvironment(): boolean { + return process.env.NODE_ENV === 'test' || Boolean(process.env.VITEST) || Boolean(process.env.VITEST_WORKER_ID); + } + + private static scheduleFlush(): void { + if (this.isTestEnvironment()) { + void this.flush(); + return; + } + + if (!this.flushTimer) { + this.flushTimer = setTimeout(() => { + this.flushTimer = null; + void this.flush(); + }, DEFAULT_FLUSH_INTERVAL_MS); + if (this.flushTimer.unref) { + this.flushTimer.unref(); + } + } + } + + /** + * Record an incoming API request against an API key in memory. + */ + public static recordRequest(keyId: string): void { + if (!keyId) return; + + if (this.isTestEnvironment()) { + void api_key_model.updateOne( + { key_id: keyId }, + { + $set: { 'usage.last_used_at': new Date() }, + $inc: { 'usage.total_requests': 1 }, + } + ).catch(() => {}); + return; + } + + const entry = this.getOrCreate(keyId); + entry.total_requests += 1; + entry.last_used_at = new Date(); + + if (entry.total_requests >= DEFAULT_MAX_BATCH_THRESHOLD) { + void this.flush(); + } else { + this.scheduleFlush(); + } + } + + /** + * Record processed notifications against an API key in memory. + */ + public static recordNotifications( + keyId: string, + channelCounts: Record, + createdCount: number + ): void { + if (!keyId || createdCount <= 0) return; + + if (this.isTestEnvironment()) { + const incObj: Record = { + 'usage.total_notifications': createdCount, + }; + for (const [channel, count] of Object.entries(channelCounts)) { + incObj[`usage.by_channel.${channel}`] = count; + } + + void api_key_model.updateOne( + { key_id: keyId }, + { + $set: { 'usage.last_used_at': new Date() }, + $inc: incObj, + } + ).catch((err: unknown) => { + logger.warn(`Failed to update usage for API key ${keyId}`, { + error: err instanceof Error ? err.message : String(err), + }); + }); + return; + } + + const entry = this.getOrCreate(keyId); + entry.total_notifications += createdCount; + entry.last_used_at = new Date(); + + for (const [channel, count] of Object.entries(channelCounts)) { + entry.by_channel[channel] = (entry.by_channel[channel] || 0) + count; + } + + if (entry.total_notifications >= DEFAULT_MAX_BATCH_THRESHOLD) { + void this.flush(); + } else { + this.scheduleFlush(); + } + } + + /** + * Flush all buffered usage statistics atomically to MongoDB. + */ + public static async flush(): Promise { + if (this.isFlushing || this.pendingUsage.size === 0) { + return; + } + + if (this.flushTimer) { + clearTimeout(this.flushTimer); + this.flushTimer = null; + } + + this.isFlushing = true; + + try { + while (this.pendingUsage.size > 0) { + const snapshot = new Map(this.pendingUsage); + this.pendingUsage.clear(); + + const updates: Promise[] = []; + + for (const [keyId, usage] of snapshot.entries()) { + const incObj: Record = {}; + + if (usage.total_requests > 0) { + incObj['usage.total_requests'] = usage.total_requests; + } + if (usage.total_notifications > 0) { + incObj['usage.total_notifications'] = usage.total_notifications; + } + for (const [ch, count] of Object.entries(usage.by_channel)) { + if (count > 0) { + incObj[`usage.by_channel.${ch}`] = count; + } + } + + if (Object.keys(incObj).length > 0) { + updates.push( + api_key_model.updateOne( + { key_id: keyId }, + { + $set: { 'usage.last_used_at': usage.last_used_at }, + $inc: incObj, + } + ).catch((err: unknown) => { + logger.warn(`Failed to flush buffered usage for API key ${keyId}`, { + error: err instanceof Error ? err.message : String(err), + }); + }) + ); + } + } + + await Promise.all(updates); + } + } catch (err) { + logger.error('Error during API key usage flush', err); + } finally { + this.isFlushing = false; + } + } +} diff --git a/src/api/utils/utils.ts b/src/api/utils/utils.ts index 1b6c7b2..a24025c 100644 --- a/src/api/utils/utils.ts +++ b/src/api/utils/utils.ts @@ -20,7 +20,7 @@ import { NOTIFICATION_STATUS, OUTBOX_STATUS } from "@src/types/types.js"; import mongoose from "mongoose"; import notification_model from "@src/database/models/notification.models.js"; import outbox_model from "@src/database/models/outbox.models.js"; -import api_key_model from "@src/database/models/api-key.models.js"; +import { ApiKeyUsageAggregator } from "./api-key-cache.js"; import { apiLogger as logger } from "@src/workers/utils/logger.js"; import { PluginRegistry } from "@src/plugins/index.js"; import notification_template_model from "@src/database/models/notification-template.models.js"; @@ -416,12 +416,16 @@ export const process_notifications = async ( const outbox_entries: outbox[] = []; const duplicate_keys: { request_id: string; channel: string }[] = []; - for (const notification of notifications) { - // Check for duplicates (include session for transactional read) - const existingNotification = await notification_model - .findOne({ - request_id: notification.request_id, - channel: notification.channel, + const existingDuplicateSet = new Set(); + + if (notifications.length > 1) { + const pairs = notifications.map((n) => ({ + request_id: n.request_id, + channel: n.channel, + })); + const existingDocs = await notification_model + .find({ + $or: pairs, status: { $in: [ NOTIFICATION_STATUS.pending, @@ -430,9 +434,47 @@ export const process_notifications = async ( ], }, }) - .session(session); + .select('request_id channel') + .session(session) + .lean(); + + for (const doc of existingDocs) { + existingDuplicateSet.add(`${doc.request_id}:${doc.channel}`); + } + } - if (existingNotification) { + for (const notification of notifications) { + let isDuplicate = false; + + if (notifications.length > 1) { + isDuplicate = existingDuplicateSet.has( + `${notification.request_id}:${notification.channel}`, + ); + } else { + const query = notification_model + .findOne({ + request_id: notification.request_id, + channel: notification.channel, + status: { + $in: [ + NOTIFICATION_STATUS.pending, + NOTIFICATION_STATUS.processing, + NOTIFICATION_STATUS.delivered, + ], + }, + }) + .session(session); + + const existingNotification = + typeof (query as unknown as { select?: (fields: string) => { lean: () => Promise } }).select === 'function' + ? await (query as unknown as { select: (fields: string) => { lean: () => Promise } }) + .select('_id') + .lean() + : await query; + isDuplicate = Boolean(existingNotification); + } + + if (isDuplicate) { duplicate_keys.push({ request_id: notification.request_id as string, channel: notification.channel, @@ -468,14 +510,12 @@ export const process_notifications = async ( ); } - // Insert notifications in bulk within transaction + // Insert notifications and outbox entries concurrently within transaction if (notifications_to_insert.length > 0) { - await notification_model.insertMany(notifications_to_insert, { session }); - } - - // Insert outbox entries in bulk within transaction - if (outbox_entries.length > 0) { - await outbox_model.insertMany(outbox_entries, { session }); + await Promise.all([ + notification_model.insertMany(notifications_to_insert, { session, ordered: false }), + outbox_model.insertMany(outbox_entries, { session, ordered: false }), + ]); } // Commit transaction - both succeed or both fail @@ -572,22 +612,5 @@ export const updateApiKeyNotificationUsage = ( channelCounts[n.channel] = (channelCounts[n.channel] || 0) + 1; } - const incObj: Record = { - 'usage.total_notifications': createdCount, - }; - for (const [ch, count] of Object.entries(channelCounts)) { - incObj[`usage.by_channel.${ch}`] = count; - } - - void api_key_model.updateOne( - { key_id: apiKey.key_id }, - { - $set: { 'usage.last_used_at': new Date() }, - $inc: incObj, - }, - ).catch((err: unknown) => - logger.warn('Failed to update API key notification usage', { - error: String(err), - }), - ); + ApiKeyUsageAggregator.recordNotifications(apiKey.key_id, channelCounts, createdCount); }; diff --git a/src/config/db.config.ts b/src/config/db.config.ts index 878cfaa..0a15103 100644 --- a/src/config/db.config.ts +++ b/src/config/db.config.ts @@ -6,10 +6,10 @@ import { AdminAlertService } from "@src/admin-alerts/admin-alert.service.js"; export const connectMongoDB = async ()=>{ try{ const db = await mongoose.connect(env.MONGO_URI, { - maxPoolSize: 200, + maxPoolSize: 300, minPoolSize: 20, - maxConnecting: 10, - serverSelectionTimeoutMS: 5000, + maxConnecting: 20, + serverSelectionTimeoutMS: 10000, }); await db.connection.syncIndexes(); logger.info("MongoDB indexes synced"); diff --git a/src/types/types.ts b/src/types/types.ts index 9c688bb..563e518 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -616,3 +616,15 @@ export type CreateApiKeyInput = z.infer; export type ApiKeyResponse = z.infer; export type CreateApiKeyResponse = z.infer; +export interface ApiKeyCacheEntry { + keyDoc: ApiKeyDoc; + cachedAt: number; +} + +export interface PendingApiKeyUsage { + total_requests: number; + total_notifications: number; + by_channel: Record; + last_used_at: Date; +} + From 56a81463e4680cf171e1e6d230cace74c8046d7b Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 4 Oct 2026 19:11:11 +0530 Subject: [PATCH 36/46] feat: remove deprecated authentication routes and implement webhook handling --- dashboard/app/api/auth/login/route.ts | 60 -------------- dashboard/app/api/auth/logout/route.ts | 27 ------- dashboard/app/api/auth/setup/route.ts | 78 ------------------- dashboard/app/api/auth/status/route.ts | 18 ----- dashboard/app/api/runtime-config/route.ts | 14 ---- dashboard/app/api/webhook/route.ts | 26 ------- .../send/batch-notification-form.tsx | 6 +- .../send/single-notification-form.tsx | 6 +- dashboard/lib/api-config.ts | 10 ++- dashboard/middleware.ts | 75 ++++++------------ src/api/controllers/admin-auth.controller.ts | 2 + src/api/controllers/webhook.controller.ts | 27 +++++++ src/api/middlewares/auth_middleware.ts | 3 +- src/api/routes/admin-auth.routes.ts | 3 +- src/api/routes/webhook.routes.ts | 12 +++ src/api/server.ts | 50 +++++++----- src/config/env.config.ts | 3 + tests/unit/api/webhook.controller.test.ts | 40 ++++++++++ 18 files changed, 160 insertions(+), 300 deletions(-) delete mode 100644 dashboard/app/api/auth/login/route.ts delete mode 100644 dashboard/app/api/auth/logout/route.ts delete mode 100644 dashboard/app/api/auth/setup/route.ts delete mode 100644 dashboard/app/api/auth/status/route.ts delete mode 100644 dashboard/app/api/runtime-config/route.ts delete mode 100644 dashboard/app/api/webhook/route.ts create mode 100644 src/api/controllers/webhook.controller.ts create mode 100644 src/api/routes/webhook.routes.ts create mode 100644 tests/unit/api/webhook.controller.test.ts diff --git a/dashboard/app/api/auth/login/route.ts b/dashboard/app/api/auth/login/route.ts deleted file mode 100644 index daacb39..0000000 --- a/dashboard/app/api/auth/login/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { NextResponse } from "next/server"; -import axios from "axios"; -import { getBasePath } from "@/lib/utils"; -import { API_BASE_URL } from "@/lib/api-config"; - -export async function POST(request: Request) { - try { - const body = await request.json(); - const { username, password } = body; - - if (!username || !password) { - return NextResponse.json( - { error: "Username and password are required" }, - { status: 400 } - ); - } - - const response = await axios.post( - `${API_BASE_URL}/api/admin/auth/login`, - { username, password }, - { - headers: { "Content-Type": "application/json" }, - timeout: 5000, - } - ); - - const basePath = getBasePath(); - const nextRes = NextResponse.json({ - success: true, - redirectUrl: `${basePath}/dashboard`, - user: response.data?.user, - }); - - const token = response.data?.token; - if (token) { - nextRes.cookies.set("simplens_session", token, { - httpOnly: true, - sameSite: "lax", - secure: process.env.NODE_ENV === "production", - maxAge: 7 * 24 * 60 * 60, - path: "/", - }); - } - - return nextRes; - } catch (error: unknown) { - if (axios.isAxiosError(error) && error.response) { - const data = error.response.data as { message?: string; error?: string }; - return NextResponse.json( - { error: data?.message || data?.error || "Invalid username or password" }, - { status: error.response.status } - ); - } - console.error("Login error:", error); - return NextResponse.json( - { error: "An error occurred during login" }, - { status: 500 } - ); - } -} diff --git a/dashboard/app/api/auth/logout/route.ts b/dashboard/app/api/auth/logout/route.ts deleted file mode 100644 index c31cc8c..0000000 --- a/dashboard/app/api/auth/logout/route.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { NextResponse } from "next/server"; -import { cookies } from "next/headers"; -import axios from "axios"; -import { API_BASE_URL } from "@/lib/api-config"; - -export async function POST() { - try { - const cookieStore = await cookies(); - cookieStore.delete("simplens_session"); - - try { - await axios.post(`${API_BASE_URL}/api/admin/auth/logout`, {}, { timeout: 3000 }); - } catch { - // Ignore backend errors on logout - } - - const res = NextResponse.json({ success: true }); - res.cookies.delete("simplens_session"); - return res; - } catch (error) { - console.error("Logout error:", error); - return NextResponse.json( - { error: "An error occurred during logout" }, - { status: 500 } - ); - } -} diff --git a/dashboard/app/api/auth/setup/route.ts b/dashboard/app/api/auth/setup/route.ts deleted file mode 100644 index 4a534a5..0000000 --- a/dashboard/app/api/auth/setup/route.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { NextResponse } from "next/server"; -import axios from "axios"; -import { API_BASE_URL } from "@/lib/api-config"; -import { getBasePath } from "@/lib/utils"; - -export async function POST(request: Request) { - try { - const body = await request.json(); - const { username, password } = body; - - if (!username || !password) { - return NextResponse.json( - { error: "Username and password are required" }, - { status: 400 } - ); - } - - const trimmedUsername = String(username).trim(); - if (trimmedUsername.length < 3 || trimmedUsername.length > 50) { - return NextResponse.json( - { error: "Username must be between 3 and 50 characters" }, - { status: 400 } - ); - } - - if (password.length < 8 || password.length > 64) { - return NextResponse.json( - { error: "Password must be between 8 and 64 characters" }, - { status: 400 } - ); - } - - const response = await axios.post( - `${API_BASE_URL}/api/admin/auth/signup`, - { username, password }, - { - headers: { - "Content-Type": "application/json", - }, - timeout: 5000, - } - ); - - const basePath = getBasePath(); - const nextRes = NextResponse.json({ - success: true, - redirectUrl: `${basePath}/dashboard`, - user: response.data?.user, - }); - - const token = response.data?.token; - if (token) { - nextRes.cookies.set("simplens_session", token, { - httpOnly: true, - sameSite: "lax", - secure: process.env.NODE_ENV === "production", - maxAge: 7 * 24 * 60 * 60, - path: "/", - }); - } - - return nextRes; - } catch (error: unknown) { - console.error("Admin setup error:", error); - if (axios.isAxiosError(error) && error.response?.data) { - const data = error.response.data as { error?: string; details?: unknown }; - return NextResponse.json( - { error: data.error || "Setup failed", details: data.details }, - { status: error.response.status || 400 } - ); - } - - return NextResponse.json( - { error: "An unexpected error occurred during setup" }, - { status: 500 } - ); - } -} diff --git a/dashboard/app/api/auth/status/route.ts b/dashboard/app/api/auth/status/route.ts deleted file mode 100644 index 36dad79..0000000 --- a/dashboard/app/api/auth/status/route.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { NextResponse } from "next/server"; -import axios from "axios"; -import { API_BASE_URL } from "@/lib/api-config"; - -export async function GET() { - try { - const response = await axios.get(`${API_BASE_URL}/api/admin/auth/status`, { - timeout: 5000, - }); - - return NextResponse.json(response.data); - } catch (error) { - console.error("Error checking auth status from backend:", error); - // Fallback check against process.env - const hasEnv = Boolean(process.env.ADMIN_USERNAME && process.env.ADMIN_PASSWORD); - return NextResponse.json({ isConfigured: hasEnv }); - } -} diff --git a/dashboard/app/api/runtime-config/route.ts b/dashboard/app/api/runtime-config/route.ts deleted file mode 100644 index e2872ec..0000000 --- a/dashboard/app/api/runtime-config/route.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { NextResponse } from "next/server"; - -/** - * Runtime configuration API endpoint - * Returns the base path configured at runtime via environment variable - * This serves as a fallback if /runtime-config.js is not loaded - */ -export async function GET() { - return NextResponse.json({ - basePath: process.env.BASE_PATH || "", - }); -} - -export const dynamic = 'force-dynamic'; diff --git a/dashboard/app/api/webhook/route.ts b/dashboard/app/api/webhook/route.ts deleted file mode 100644 index 06c29e7..0000000 --- a/dashboard/app/api/webhook/route.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { NextResponse } from "next/server"; - -/** - * Webhook endpoint to receive notification delivery status callbacks. - * The worker already updates status in MongoDB, so we just acknowledge receipt. - */ -export async function POST(request: Request) { - try { - const payload = await request.json(); - - // Log the webhook callback for debugging (optional) - console.log("[Webhook] Received callback:", { - request_id: payload.request_id, - notification_id: payload.notification_id, - status: payload.status, - channel: payload.channel, - occurred_at: payload.occurred_at, - }); - - // Return 200 OK to acknowledge receipt - return NextResponse.json({ received: true }); - } catch (error) { - console.error("[Webhook] Error processing callback:", error); - return NextResponse.json({ received: false, error: "Invalid payload" }, { status: 400 }); - } -} diff --git a/dashboard/components/send/batch-notification-form.tsx b/dashboard/components/send/batch-notification-form.tsx index 7c56669..20cdd58 100644 --- a/dashboard/components/send/batch-notification-form.tsx +++ b/dashboard/components/send/batch-notification-form.tsx @@ -16,6 +16,7 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { pluginService, templateService, notificationService } from "@/lib/api-client"; +import { withBasePath } from "@/lib/utils"; // Generate a UUIDv4 function generateUUID(): string { @@ -400,8 +401,9 @@ export function BatchNotificationForm({ onSuccess }: BatchNotificationFormProps) client_id: clientId, channel: selectedChannels, recipients: processedRecipients, - //hard-coded the webhook url as the simplest solution. This url may not be correct all the times. Nop issues, the worker just fire and forgets while sending webhook, it does not retires - webhook_url: "http://localhost:3002/api/webhook" + webhook_url: typeof window !== "undefined" + ? `${window.location.origin}${withBasePath("/api/webhook")}` + : "http://localhost:3000/api/webhook", }; const hasAnyTemplate = selectedChannels.some( diff --git a/dashboard/components/send/single-notification-form.tsx b/dashboard/components/send/single-notification-form.tsx index 1346c97..8adbe5a 100644 --- a/dashboard/components/send/single-notification-form.tsx +++ b/dashboard/components/send/single-notification-form.tsx @@ -16,6 +16,7 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { pluginService, templateService, notificationService } from "@/lib/api-client"; +import { withBasePath } from "@/lib/utils"; // Generate a UUIDv4 function generateUUID(): string { @@ -324,8 +325,9 @@ export function SingleNotificationForm({ onSuccess }: SingleNotificationFormProp client_id: clientId, channel: selectedChannels, recipient: { ...recipientData }, - //hard-coded the webhook url as the simplest solution. This url may not be correct all the times. Nop issues, the worker just fire and forgets while sending webhook, it does not retires - webhook_url: "http://localhost:3002/api/webhook" + webhook_url: typeof window !== "undefined" + ? `${window.location.origin}${withBasePath("/api/webhook")}` + : "http://localhost:3000/api/webhook", }; const hasAnyTemplate = selectedChannels.some( diff --git a/dashboard/lib/api-config.ts b/dashboard/lib/api-config.ts index 7fc6641..e985e51 100644 --- a/dashboard/lib/api-config.ts +++ b/dashboard/lib/api-config.ts @@ -1,6 +1,12 @@ /** * API Configuration - * Centralized configuration for API endpoints + * Centralized configuration for backend API base URL resolution */ -export const API_BASE_URL = process.env.API_BASE_URL || 'http://localhost:3000'; \ No newline at end of file +const rawBaseUrl = (process.env.API_BASE_URL || 'http://localhost:3000').trim().replace(/\/+$/, ''); +const rawBasePath = (process.env.BASE_PATH || '').trim().replace(/\/+$/, ''); +const normalizedBasePath = rawBasePath.startsWith('/') ? rawBasePath : rawBasePath ? `/${rawBasePath}` : ''; + +export const API_BASE_URL = normalizedBasePath && !rawBaseUrl.endsWith(normalizedBasePath) + ? `${rawBaseUrl}${normalizedBasePath}` + : rawBaseUrl; diff --git a/dashboard/middleware.ts b/dashboard/middleware.ts index a0db2a7..ae9f026 100644 --- a/dashboard/middleware.ts +++ b/dashboard/middleware.ts @@ -5,16 +5,6 @@ import { API_BASE_URL } from "./lib/api-config"; const basePath = process.env.BASE_PATH || ""; const SESSION_COOKIE_NAME = "simplens_session"; -const localApiPaths = [ - "/api/auth/", - "/api/runtime-config", - "/api/webhook" -]; - -function isLocalApiRoute(pathname: string): boolean { - return localApiPaths.some((p) => pathname.startsWith(p)); -} - function proxyToBackend(request: NextRequest, pathname: string, jwtToken?: string) { const requestHeaders = new Headers(request.headers); if (jwtToken) { @@ -33,7 +23,6 @@ function proxyToBackend(request: NextRequest, pathname: string, jwtToken?: strin const publicRoutes = [ "/login", "/setup", - "/api/admin/auth", "/api/auth/login", "/api/auth/setup", "/api/auth/status", @@ -67,13 +56,10 @@ function validateSessionFromRequest(request: NextRequest): { } function isPublicRoute(pathname: string): boolean { - // Remove base path prefix if present let normalizedPath = pathname; if (basePath && pathname.startsWith(basePath)) { normalizedPath = pathname.slice(basePath.length) || "/"; } - - // Check if it's a public route return publicRoutes.some((route) => normalizedPath.startsWith(route)); } @@ -85,7 +71,6 @@ export async function middleware(request: NextRequest) { let pathname = request.nextUrl.pathname; // STEP 0: Handle root path when base path is configured - // Redirect "/" to basePath (e.g., "/dashboard") if (basePath && pathname === "/" && basePath !== "") { return NextResponse.redirect(new URL(basePath, request.url)); } @@ -93,11 +78,10 @@ export async function middleware(request: NextRequest) { // STEP 1: Strip base path if present and rewrite the request if (basePath && pathname.startsWith(basePath)) { const strippedPath = pathname.slice(basePath.length) || '/'; - const url = request.nextUrl.clone(); url.pathname = strippedPath; pathname = strippedPath; - + if (isStaticFile(pathname)) { return NextResponse.rewrite(url); } @@ -124,12 +108,31 @@ export async function middleware(request: NextRequest) { return NextResponse.redirect(new URL(`${basePath}/dashboard`, request.url)); } - // STEP 4.5: Proxy public backend auth routes directly to API server - if (pathname.startsWith("/api/admin/auth")) { - return proxyToBackend(request, pathname); + // STEP 5: Proxy API requests directly to backend when accessed on standalone dashboard port + if (pathname.startsWith("/api/")) { + // Public API routes + if (isPublicRoute(pathname)) { + return proxyToBackend(request, pathname); + } + + // Authenticated session + if (session.isValid) { + return proxyToBackend(request, pathname, session.token); + } + + // Bearer token fallback for external/MCP clients + const authHeader = request.headers.get("authorization"); + if (authHeader && authHeader.startsWith("Bearer ")) { + return proxyToBackend(request, pathname); + } + + return NextResponse.json( + { error: "Unauthorized: valid session or API key required" }, + { status: 401 } + ); } - // STEP 5: Allow public routes + // STEP 6: Allow public UI pages if (isPublicRoute(pathname)) { if (basePath && request.nextUrl.pathname.startsWith(basePath)) { const url = request.nextUrl.clone(); @@ -139,41 +142,13 @@ export async function middleware(request: NextRequest) { return NextResponse.next(); } - // STEP 6: Protected routes + // STEP 7: Protected UI pages - redirect unauthenticated users to login if (!session.isValid) { - // For API routes, fall back to Authorization header (API key or Bearer token) - // passed by external services or MCP server directly through the dashboard - if (pathname.startsWith("/api/")) { - const authHeader = request.headers.get("authorization"); - if (authHeader && authHeader.startsWith("Bearer ")) { - if (isLocalApiRoute(pathname)) { - if (basePath && request.nextUrl.pathname.startsWith(basePath)) { - const url = request.nextUrl.clone(); - url.pathname = pathname; - return NextResponse.rewrite(url); - } - return NextResponse.next(); - } - return proxyToBackend(request, pathname); - } - - return NextResponse.json( - { error: "Unauthorized: valid session or API key required" }, - { status: 401 } - ); - } - - // Non-API routes: redirect to login const loginUrl = new URL(`${basePath}/login`, request.url); loginUrl.searchParams.set("callbackUrl", request.nextUrl.pathname); return NextResponse.redirect(loginUrl); } - // STEP 7: Proxy non-local API requests attaching the user's Core API JWT token - if (pathname.startsWith("/api/") && !isLocalApiRoute(pathname)) { - return proxyToBackend(request, pathname, session.token); - } - if (basePath && request.nextUrl.pathname.startsWith(basePath)) { const url = request.nextUrl.clone(); url.pathname = pathname; diff --git a/src/api/controllers/admin-auth.controller.ts b/src/api/controllers/admin-auth.controller.ts index 5e3c98a..f4dc32e 100644 --- a/src/api/controllers/admin-auth.controller.ts +++ b/src/api/controllers/admin-auth.controller.ts @@ -141,6 +141,7 @@ export const verifyAdmin = async (req: Request, res: Response): Promise => const token = generateAdminJwt(user); setSessionCookie(res, token); res.status(200).json({ + success: true, isValid: true, token, user, @@ -160,6 +161,7 @@ export const verifyAdmin = async (req: Request, res: Response): Promise => const token = generateAdminJwt(user); setSessionCookie(res, token); res.status(200).json({ + success: true, isValid: true, token, user, diff --git a/src/api/controllers/webhook.controller.ts b/src/api/controllers/webhook.controller.ts new file mode 100644 index 0000000..e3977e9 --- /dev/null +++ b/src/api/controllers/webhook.controller.ts @@ -0,0 +1,27 @@ +/** + * Webhook Controller + * + * Handles incoming notification status callbacks (e.g. delivery receipts, provider webhooks). + */ + +import type { Request, Response } from 'express'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +export const handleWebhookCallback = async (req: Request, res: Response): Promise => { + try { + const payload = req.body; + + logger.info('[Webhook] Received callback:', { + request_id: payload?.request_id, + notification_id: payload?.notification_id, + status: payload?.status, + channel: payload?.channel, + occurred_at: payload?.occurred_at, + }); + + res.status(200).json({ received: true }); + } catch (error) { + logger.error('[Webhook] Error processing callback:', error); + res.status(400).json({ received: false, error: 'Invalid payload' }); + } +}; diff --git a/src/api/middlewares/auth_middleware.ts b/src/api/middlewares/auth_middleware.ts index 5141377..bc5b02e 100644 --- a/src/api/middlewares/auth_middleware.ts +++ b/src/api/middlewares/auth_middleware.ts @@ -55,7 +55,8 @@ async function handleDynamicApiKey( } const fullPath = (req.originalUrl || `${req.baseUrl || ''}${req.path}`).split('?')[0]; - if (fullPath.includes('/api/admin/auth/setup') || fullPath.includes('/api/admin/auth/verify')) { + const isAdminCredentialRoute = /(?:\/api)?(?:\/admin)?\/auth\/(?:setup|signup|login|verify)/.test(fullPath); + if (isAdminCredentialRoute) { res.status(403).json({ error: 'Forbidden: API keys cannot access administrator credential setup or verification routes.', }); diff --git a/src/api/routes/admin-auth.routes.ts b/src/api/routes/admin-auth.routes.ts index b690bef..9f4fc63 100644 --- a/src/api/routes/admin-auth.routes.ts +++ b/src/api/routes/admin-auth.routes.ts @@ -12,8 +12,9 @@ import { const router = Router(); -// 1. Signup - Initial administrator account creation (only allowed if no admin exists) +// 1. Signup / Setup - Initial administrator account creation (only allowed if no admin exists) router.post('/signup', signupAdmin); +router.post('/setup', signupAdmin); // 2. Login - Verify administrator credentials and set session cookie router.post('/login', loginAdmin); diff --git a/src/api/routes/webhook.routes.ts b/src/api/routes/webhook.routes.ts new file mode 100644 index 0000000..b8d9765 --- /dev/null +++ b/src/api/routes/webhook.routes.ts @@ -0,0 +1,12 @@ +/** + * Webhook Routes + */ + +import { Router } from 'express'; +import { handleWebhookCallback } from '../controllers/webhook.controller.js'; + +const router = Router(); + +router.post('/', handleWebhookCallback); + +export default router; diff --git a/src/api/server.ts b/src/api/server.ts index 4c3f722..aa5c076 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -9,12 +9,13 @@ import plugins_router from './routes/plugins.routes.js'; import providers_router from './routes/providers.routes.js'; import channel_routing_router from './routes/channel-routing.routes.js'; import admin_channels_router from './routes/admin-channels.routes.js'; -import notification_templates_router from '@src/api/routes/notification_templates.routes.js'; +import notification_templates_router from './routes/notification_templates.routes.js'; import notifications_management_router from './routes/notifications-management.routes.js'; import alerts_router from './routes/alerts.routes.js'; import dashboard_router from './routes/dashboard.routes.js'; import settings_router from './routes/settings.routes.js'; import admin_auth_router from './routes/admin-auth.routes.js'; +import webhook_router from './routes/webhook.routes.js'; import api_keys_router from './routes/api-key.routes.js'; import health_router from './routes/health.routes.js'; import { @@ -63,7 +64,16 @@ app.use(cors({ credentials: true, })); -app.get("/api", (req: Request, res: Response) => { +// check whether the BASE_PATH is a valid one +let BASE_PATH = env.BASE_PATH; +if (!(BASE_PATH.length > 0 && BASE_PATH.startsWith("/"))) { + BASE_PATH = ""; +} +if (BASE_PATH.endsWith("/")) { + BASE_PATH = BASE_PATH.slice(0, BASE_PATH.length - 1); +} + +app.get(`${BASE_PATH}/api`, (req: Request, res: Response) => { res.json({ info: "Notification Service is running" }); @@ -71,20 +81,21 @@ app.get("/api", (req: Request, res: Response) => { }); // Health check endpoints for Docker/Kubernetes and API consumers -app.use('/api', health_router); - -app.use('/api/notification', auth_middleware, notification_router); -app.use('/api/notifications', auth_middleware, notifications_management_router); -app.use('/api/alerts', auth_middleware, alerts_router); -app.use('/api/dashboard', auth_middleware, dashboard_router); -app.use('/api/plugins', auth_middleware, plugins_router); -app.use('/api/providers', auth_middleware, providers_router); -app.use('/api/channels/routing', auth_middleware, channel_routing_router); -app.use('/api/admin-channels', auth_middleware, admin_channels_router); -app.use('/api/templates', auth_middleware, notification_templates_router); -app.use('/api/settings', auth_middleware, settings_router); -app.use('/api/keys', auth_middleware, api_keys_router); -app.use('/api/admin/auth', admin_auth_router); +app.use(`${BASE_PATH}/api`, health_router); + +app.use(`${BASE_PATH}/api/notification`, auth_middleware, notification_router); +app.use(`${BASE_PATH}/api/notifications`, auth_middleware, notifications_management_router); +app.use(`${BASE_PATH}/api/alerts`, auth_middleware, alerts_router); +app.use(`${BASE_PATH}/api/dashboard`, auth_middleware, dashboard_router); +app.use(`${BASE_PATH}/api/plugins`, auth_middleware, plugins_router); +app.use(`${BASE_PATH}/api/providers`, auth_middleware, providers_router); +app.use(`${BASE_PATH}/api/channels/routing`, auth_middleware, channel_routing_router); +app.use(`${BASE_PATH}/api/admin-channels`, auth_middleware, admin_channels_router); +app.use(`${BASE_PATH}/api/templates`, auth_middleware, notification_templates_router); +app.use(`${BASE_PATH}/api/settings`, auth_middleware, settings_router); +app.use(`${BASE_PATH}/api/keys`, auth_middleware, api_keys_router); +app.use(`${BASE_PATH}/api/auth`, admin_auth_router); +app.use(`${BASE_PATH}/api/webhook`, webhook_router); const start_server = async () => { try { @@ -120,7 +131,7 @@ const start_server = async () => { const server = http.createServer(app); server.keepAliveTimeout = 65000; server.headersTimeout = 66000; - server.listen(env.PORT, () => logger.success(`Notification Service running at http://localhost:${env.PORT}`)); + server.listen(env.PORT, () => logger.success(`Notification Service running at http://localhost:${env.PORT}${BASE_PATH}`)); // 5. Start health probe server for Kubernetes probeServer = createHealthProbeServer({ @@ -148,6 +159,7 @@ const start_server = async () => { }); } await ApiKeyUsageAggregator.flush(); + await PluginSyncService.stopSubscriber(); await disconnectRedis(); await db.disconnect(); } catch (e) { @@ -165,8 +177,8 @@ const start_server = async () => { void AdminAlertService.sendAlert('service_health', `🔴 UNCAUGHT EXCEPTION IN API SERVER\n` + - `Error: ${err.message}\n` + - `Stack: ${err.stack?.split('\n').slice(0, 3).join('\n')}\n` + + `Error: ${err instanceof Error ? err.message : String(err)}\n` + + `Stack: ${err?.stack?.split('\n').slice(0, 3).join('\n') || "undefined"}\n` + `Action: Review error handling. Check for async operations without try-catch.`, { severity: 'critical' }); diff --git a/src/config/env.config.ts b/src/config/env.config.ts index 9f3c07c..d2651de 100644 --- a/src/config/env.config.ts +++ b/src/config/env.config.ts @@ -71,6 +71,9 @@ export const env = { // Admin Alerts (optional - auto-generated if not provided) ADMIN_ALERT_ENCRYPTION_KEY: process.env.ADMIN_ALERT_ENCRYPTION_KEY || "", + // Application Base Path + BASE_PATH: process.env.BASE_PATH || "", + // ======================================================================== // 2. DYNAMIC OPERATIONAL VARIABLES (Delegated to DynamicConfigService) // These reflect live in-memory MongoDB configuration with zero-restart updates. diff --git a/tests/unit/api/webhook.controller.test.ts b/tests/unit/api/webhook.controller.test.ts new file mode 100644 index 0000000..14386dd --- /dev/null +++ b/tests/unit/api/webhook.controller.test.ts @@ -0,0 +1,40 @@ +import { describe, it, expect } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import webhook_router from '@src/api/routes/webhook.routes.js'; + +describe('Webhook Controller', () => { + let app: express.Application; + + beforeEach(() => { + app = express(); + app.use(express.json()); + app.use('/api/webhook', webhook_router); + }); + + it('should acknowledge valid webhook status callbacks with 200 and { received: true }', async () => { + const payload = { + request_id: 'req-123', + notification_id: 'notif-456', + status: 'delivered', + channel: 'email', + occurred_at: new Date().toISOString(), + }; + + const res = await request(app) + .post('/api/webhook') + .send(payload) + .expect(200); + + expect(res.body).toEqual({ received: true }); + }); + + it('should return 200 even for empty payload', async () => { + const res = await request(app) + .post('/api/webhook') + .send({}) + .expect(200); + + expect(res.body).toEqual({ received: true }); + }); +}); From 11c3bf1e17891091e30b76fae06a14bdee7ec75a Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 11 Oct 2026 11:57:31 +0530 Subject: [PATCH 37/46] feat: enhance nginx configuration generation and update base path prompts --- packages/onboard/src/__tests__/infra.test.ts | 61 +++++++++++++++++++- packages/onboard/src/env-config.ts | 2 +- packages/onboard/src/index.ts | 6 ++ packages/onboard/src/infra.ts | 33 +++++++++-- packages/onboard/src/templates.ts | 7 +-- 5 files changed, 99 insertions(+), 10 deletions(-) diff --git a/packages/onboard/src/__tests__/infra.test.ts b/packages/onboard/src/__tests__/infra.test.ts index ea92669..8e2d8dd 100644 --- a/packages/onboard/src/__tests__/infra.test.ts +++ b/packages/onboard/src/__tests__/infra.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { buildAppComposeContent } from '../infra.js'; +import { buildAppComposeContent, generateNginxConfig } from '../infra.js'; +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; describe('infra app compose generation', () => { it('does not include nginx when disabled', () => { @@ -52,3 +55,59 @@ describe('infra app compose generation', () => { expect(compose).toContain('logs-data:'); }); }); + +describe('generateNginxConfig', () => { + it('generates nginx config for root basePath correctly', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'onboard-test-')); + try { + await generateNginxConfig(tmpDir, ''); + const content = await fs.readFile(path.join(tmpDir, 'nginx.conf'), 'utf-8'); + expect(content).toContain('location ^~ /api {'); + expect(content).toContain('proxy_pass http://api:3000;'); + expect(content).toContain('location / {'); + expect(content).toContain('proxy_pass http://dashboard:3002;'); + expect(content).not.toContain('return 302'); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); + + it('generates nginx config for legacy version (<= 1.3.0) with custom basePath correctly', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'onboard-test-')); + try { + await generateNginxConfig(tmpDir, '/sns', { version: '1.3.0' }); + const content = await fs.readFile(path.join(tmpDir, 'nginx.conf'), 'utf-8'); + expect(content).toContain('location /api {'); + expect(content).not.toContain('location ^~ /sns/api {'); + expect(content).not.toContain('location ^~ /api {'); + expect(content).not.toContain('return 302'); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); + + it('generates nginx config for legacy version using isLegacy flag', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'onboard-test-')); + try { + await generateNginxConfig(tmpDir, '/sns', { isLegacy: true }); + const content = await fs.readFile(path.join(tmpDir, 'nginx.conf'), 'utf-8'); + expect(content).toContain('location /api {'); + expect(content).not.toContain('return 302'); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); + + it('generates nginx config for version > 1.3.0 with custom basePath correctly', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'onboard-test-')); + try { + await generateNginxConfig(tmpDir, '/sns', { version: '1.4.0' }); + const content = await fs.readFile(path.join(tmpDir, 'nginx.conf'), 'utf-8'); + expect(content).toContain('location ^~ /sns/api {'); + expect(content).toContain('return 302 /sns;'); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); +}); + diff --git a/packages/onboard/src/env-config.ts b/packages/onboard/src/env-config.ts index af64a75..be83465 100644 --- a/packages/onboard/src/env-config.ts +++ b/packages/onboard/src/env-config.ts @@ -112,7 +112,7 @@ export function normalizeBasePath(input: string): string { */ export async function promptBasePath(defaultValue: string = DEFAULT_BASE_PATH): Promise { const result = await text({ - message: 'BASE_PATH for dashboard (leave empty for root, example: /dashboard):', + message: 'BASE_PATH for API and dashboard (leave empty for root, example: /sns):', placeholder: defaultValue || 'leave empty for root', defaultValue, validate: (value: string | undefined) => { diff --git a/packages/onboard/src/index.ts b/packages/onboard/src/index.ts index 9faf05e..592569c 100644 --- a/packages/onboard/src/index.ts +++ b/packages/onboard/src/index.ts @@ -555,6 +555,8 @@ async function main() { enableSsl: setupOptions.enableSsl, domain: setupOptions.sslDomain, sslMode: setupOptions.enableSsl ? 'bootstrap' : 'final', + version: setupOptions.version, + isLegacy, }); } @@ -681,6 +683,8 @@ async function main() { enableSsl: true, domain: setupOptions.sslDomain, sslMode: 'final', + version: setupOptions.version, + isLegacy, }); await reloadNginxConfiguration(targetDir, { @@ -701,6 +705,8 @@ async function main() { domain: setupOptions.sslDomain, sslMode: 'final', filename: 'nginx.ssl.conf', + version: setupOptions.version, + isLegacy, }); // Print detailed step-by-step SSL setup instructions diff --git a/packages/onboard/src/infra.ts b/packages/onboard/src/infra.ts index 2594d67..8a0a1ef 100644 --- a/packages/onboard/src/infra.ts +++ b/packages/onboard/src/infra.ts @@ -12,6 +12,7 @@ import { multiselect } from '@clack/prompts'; import { handleCancel, spinner } from './ui.js'; import path from 'path'; import type { InfraService } from './types/domain.js'; +import { isVersionGreaterThan130 } from './env-config.js'; const INFRA_SERVICES: InfraService[] = [ { name: 'MongoDB (Database)', value: 'mongo', checked: true }, @@ -389,13 +390,22 @@ export async function writeAppCompose( s.stop('Generated docker-compose.yaml'); } +export interface NginxConfigOptions { + enableSsl?: boolean; + domain?: string; + sslMode?: 'bootstrap' | 'final'; + filename?: string; + version?: string; + isLegacy?: boolean; +} + /** * Generate nginx.conf based on basePath configuration */ export async function generateNginxConfig( targetDir: string, basePath: string, - options: { enableSsl?: boolean; domain?: string; sslMode?: 'bootstrap' | 'final'; filename?: string } = {} + options: NginxConfigOptions = {} ): Promise { const outputFilename = options.filename ?? 'nginx.conf'; const s = spinner(); @@ -407,9 +417,16 @@ export async function generateNginxConfig( const enableSsl = options.enableSsl === true; const domain = options.domain?.trim() || 'localhost'; const sslMode = options.sslMode ?? 'final'; + const isNewVersion = options.isLegacy !== undefined + ? !options.isLegacy + : isVersionGreaterThan130(options.version); + + const apiRoutePrefix = isNewVersion + ? (hasBasePath ? `^~ /${normalizedPath}/api` : '^~ /api') + : '/api'; const proxyRoutes = ` - location /api { + location ${apiRoutePrefix} { proxy_pass http://api:3000; proxy_http_version 1.1; @@ -494,6 +511,7 @@ ${hasBasePath ? ` `; const rootRedirectPath = hasBasePath ? `/${normalizedPath}` : '/'; + const shouldRedirectRoot = isNewVersion && hasBasePath; const nginxTemplate = enableSsl ? sslMode === 'bootstrap' @@ -504,7 +522,10 @@ ${hasBasePath ? ` location /.well-known/acme-challenge/ { root /var/www/certbot; } -${proxyRoutes} +${shouldRedirectRoot ? ` location = / { + return 302 ${rootRedirectPath}; + } +` : ''}${proxyRoutes} } ` : `server { @@ -536,7 +557,11 @@ ${hasBasePath ? ` location = / { ` : `server { listen 80; - server_name localhost;${proxyRoutes} + server_name localhost;${shouldRedirectRoot ? ` + location = / { + return 302 ${rootRedirectPath}; + } +` : ''}${proxyRoutes} } `; diff --git a/packages/onboard/src/templates.ts b/packages/onboard/src/templates.ts index 5b06a19..5763b8b 100644 --- a/packages/onboard/src/templates.ts +++ b/packages/onboard/src/templates.ts @@ -221,7 +221,7 @@ export const APP_COMPOSE_TEMPLATE = `services: command: [ "node", "dist/api/server.js" ] restart: unless-stopped healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] + test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000\${BASE_PATH}/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] interval: 30s timeout: 10s retries: 3 @@ -288,9 +288,8 @@ export const APP_COMPOSE_TEMPLATE = `services: - .env environment: PORT: \${DASHBOARD_PORT:-3002} - API_BASE_URL: http://api:\${PORT:-3000} - WEBHOOK_HOST: dashboard - WEBHOOK_PORT: \${DASHBOARD_PORT:-3002} + BASE_PATH: \${BASE_PATH:-} + API_BASE_URL: \${API_BASE_URL:-http://api:\${PORT:-3000}} depends_on: api: condition: service_healthy From b7e41431116381a2d60bdb4d48b064a0ca722e5e Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 11 Oct 2026 12:00:03 +0530 Subject: [PATCH 38/46] feat(docker-compose): update API_BASE_URL to include BASE_PATH for improved configuration --- docker-compose/docker-compose.dev.yaml | 5 ++--- docker-compose/docker-compose.local.yaml | 5 ++--- docker-compose/docker-compose.master.yaml | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/docker-compose/docker-compose.dev.yaml b/docker-compose/docker-compose.dev.yaml index 2a09903..c1e0052 100644 --- a/docker-compose/docker-compose.dev.yaml +++ b/docker-compose/docker-compose.dev.yaml @@ -163,11 +163,10 @@ services: environment: NODE_ENV: production PORT: ${DASHBOARD_PORT:-3002} - API_BASE_URL: http://api:${PORT:-3000} + API_BASE_URL: http://api:{PORT:-3000}{BASE_PATH:-} + BASE_PATH: ${BASE_PATH:-} MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 AUTH_TRUST_HOST: "true" - WEBHOOK_HOST: dashboard - WEBHOOK_PORT: "3002" depends_on: api: condition: service_healthy diff --git a/docker-compose/docker-compose.local.yaml b/docker-compose/docker-compose.local.yaml index e0e8398..58305c7 100644 --- a/docker-compose/docker-compose.local.yaml +++ b/docker-compose/docker-compose.local.yaml @@ -166,11 +166,10 @@ services: environment: NODE_ENV: ${NODE_ENV:-development} PORT: ${DASHBOARD_PORT:-3002} - API_BASE_URL: http://api:${PORT:-3000} + API_BASE_URL: http://api:{PORT:-3000}{BASE_PATH:-} + BASE_PATH: ${BASE_PATH:-} MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 AUTH_TRUST_HOST: "true" - WEBHOOK_HOST: dashboard - WEBHOOK_PORT: "3002" depends_on: api: condition: service_healthy diff --git a/docker-compose/docker-compose.master.yaml b/docker-compose/docker-compose.master.yaml index c09b457..fa9d965 100644 --- a/docker-compose/docker-compose.master.yaml +++ b/docker-compose/docker-compose.master.yaml @@ -163,11 +163,10 @@ services: environment: NODE_ENV: production PORT: ${DASHBOARD_PORT:-3002} - API_BASE_URL: http://api:${PORT:-3000} + API_BASE_PATH_URL: http://api:{PORT:-3000}{BASE_PATH:-} + BASE_PATH: ${BASE_PATH:-} MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 AUTH_TRUST_HOST: "true" - WEBHOOK_HOST: dashboard - WEBHOOK_PORT: "3002" depends_on: api: condition: service_healthy From 4354ae3f74d8251477fad1c98a169b4cbdab24b4 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 11 Oct 2026 12:10:14 +0530 Subject: [PATCH 39/46] fix: update alert messages for clarity in stats and notification components --- dashboard/components/dashboard/stats-cards.tsx | 2 +- dashboard/components/plugins/providers-tab.tsx | 4 ++-- dashboard/components/send/single-notification-form.tsx | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboard/components/dashboard/stats-cards.tsx b/dashboard/components/dashboard/stats-cards.tsx index e4e34fd..559d984 100644 --- a/dashboard/components/dashboard/stats-cards.tsx +++ b/dashboard/components/dashboard/stats-cards.tsx @@ -337,7 +337,7 @@ export function StatsGrid() {

No channels active yet.

-

Install plugins to see channel metrics.

+

Send channel-specific notifications to see channel metrics.

)} diff --git a/dashboard/components/plugins/providers-tab.tsx b/dashboard/components/plugins/providers-tab.tsx index 3725da0..abd7c3f 100644 --- a/dashboard/components/plugins/providers-tab.tsx +++ b/dashboard/components/plugins/providers-tab.tsx @@ -215,7 +215,7 @@ function ProviderRateLimitRing({
{status.is_exhausted ? (
- + Next token in {(status.resets_in_ms / 1000).toFixed(1)}s
) : remaining < maxTokens ? ( @@ -624,7 +624,7 @@ export function ProvidersTab() { {p.plugin_name}

-
+
handleResetRateLimit(p.id)} diff --git a/dashboard/components/send/single-notification-form.tsx b/dashboard/components/send/single-notification-form.tsx index 8adbe5a..f00130c 100644 --- a/dashboard/components/send/single-notification-form.tsx +++ b/dashboard/components/send/single-notification-form.tsx @@ -439,7 +439,7 @@ export function SingleNotificationForm({ onSuccess }: SingleNotificationFormProp No Plugins Found - No notification plugins are configured. Please install and configure plugins in simplens.config.yaml. + No notification plugins are configured. Please install and configure plugins. ); From d38b26eb25e539da53462a90d6d39635b87bcfb6 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 11 Oct 2026 12:18:52 +0530 Subject: [PATCH 40/46] 1.4.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 44716d2..bcc3040 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "simplens-core", - "version": "1.3.0", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "simplens-core", - "version": "1.3.0", + "version": "1.4.0", "license": "ISC", "dependencies": { "axios": "^1.13.2", diff --git a/package.json b/package.json index bf6ac85..6819c22 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "simplens-core", - "version": "1.3.0", + "version": "1.4.0", "private": true, "description": "Open-source plugin-based notification orchestration engine for developers who value control ", "main": "dist/api/server.js", From c160f5e9c873595cef998b4dcdafbe39b8c6b4d4 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 11 Oct 2026 12:20:22 +0530 Subject: [PATCH 41/46] chore: update dashboard version to 1.4.0 --- dashboard/package-lock.json | 4 ++-- dashboard/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 47aece9..19a4ad0 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -1,12 +1,12 @@ { "name": "dashboard", - "version": "1.3.0", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dashboard", - "version": "1.3.0", + "version": "1.4.0", "dependencies": { "@monaco-editor/react": "^4.7.0", "@radix-ui/react-alert-dialog": "^1.1.15", diff --git a/dashboard/package.json b/dashboard/package.json index fec18b4..1f21730 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "dashboard", - "version": "1.3.0", + "version": "1.4.0", "private": true, "engines": { "node": ">=22.0.0" From 1259616189e9ecc9c662c18d277bb2ce3dd69729 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 18 Oct 2026 18:48:56 +0530 Subject: [PATCH 42/46] fix: update healthcheck URLs in docker-compose files to include BASE_PATH for improved flexibility --- docker-compose/docker-compose.dev.yaml | 2 +- docker-compose/docker-compose.local.yaml | 2 +- docker-compose/docker-compose.master.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose/docker-compose.dev.yaml b/docker-compose/docker-compose.dev.yaml index c1e0052..b007df1 100644 --- a/docker-compose/docker-compose.dev.yaml +++ b/docker-compose/docker-compose.dev.yaml @@ -33,7 +33,7 @@ services: - simplens-infra restart: unless-stopped healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] + test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000${BASE_PATH:-}/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] interval: 15s timeout: 5s retries: 3 diff --git a/docker-compose/docker-compose.local.yaml b/docker-compose/docker-compose.local.yaml index 58305c7..c12f065 100644 --- a/docker-compose/docker-compose.local.yaml +++ b/docker-compose/docker-compose.local.yaml @@ -35,7 +35,7 @@ services: - simplens-infra restart: unless-stopped healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] + test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000${BASE_PATH:-}/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] interval: 15s timeout: 5s retries: 3 diff --git a/docker-compose/docker-compose.master.yaml b/docker-compose/docker-compose.master.yaml index fa9d965..b5a2fb6 100644 --- a/docker-compose/docker-compose.master.yaml +++ b/docker-compose/docker-compose.master.yaml @@ -33,7 +33,7 @@ services: - simplens-infra restart: unless-stopped healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] + test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000${BASE_PATH:-}/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] interval: 15s timeout: 5s retries: 3 @@ -163,7 +163,7 @@ services: environment: NODE_ENV: production PORT: ${DASHBOARD_PORT:-3002} - API_BASE_PATH_URL: http://api:{PORT:-3000}{BASE_PATH:-} + API_BASE_URL: http://api:{PORT:-3000}{BASE_PATH:-} BASE_PATH: ${BASE_PATH:-} MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 AUTH_TRUST_HOST: "true" From 22d4e7912578f3ae638a61107f1bbbc221b18cc8 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 18 Oct 2026 18:49:15 +0530 Subject: [PATCH 43/46] feat(types): add NPM authentication types for improved registry management --- src/plugins/services/npm-auth.service.ts | 27 +++--------------------- src/types/types.ts | 26 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/plugins/services/npm-auth.service.ts b/src/plugins/services/npm-auth.service.ts index ae34cd5..d5d8b49 100644 --- a/src/plugins/services/npm-auth.service.ts +++ b/src/plugins/services/npm-auth.service.ts @@ -13,38 +13,17 @@ import { encrypt, decrypt } from '@src/utils/encryption.utils.js'; import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; import { PLUGINS_DIR, initPluginsDir } from '@src/plugins/loader/plugin-fs.js'; import type { - encrypted_config, npm_auth_status, npm_registry_status, + StoredNpmAuthConfig, + StoredRegistryEntry, + DecryptedRegistryEntry } from '@src/types/types.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; export const NPM_AUTH_CONFIG_KEY = 'npm_auth_config'; export const NPMRC_PATH = join(PLUGINS_DIR, '.npmrc'); -interface StoredRegistryEntry { - id: string; - scope?: string; - registry_url: string; - encrypted_token: encrypted_config; - updated_at: string; -} - -interface StoredNpmAuthConfig { - registries?: StoredRegistryEntry[]; - // Legacy fields for backwards compatibility - encrypted_token?: encrypted_config; - registry_url?: string; -} - -export interface DecryptedRegistryEntry { - id: string; - scope?: string; - registry_url: string; - token: string; - updated_at: string; -} - /** * Extract npm scope from a package name if present * E.g. "@simplens/resend" -> "@simplens", "lodash" -> undefined diff --git a/src/types/types.ts b/src/types/types.ts index 563e518..a3eaa27 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -628,3 +628,29 @@ export interface PendingApiKeyUsage { last_used_at: Date; } +// ============================================================================ +// NPM AUTH TYPES +// ============================================================================ + +export interface StoredRegistryEntry { + id: string; + scope?: string; + registry_url: string; + encrypted_token: encrypted_config; + updated_at: string; +} + +export interface StoredNpmAuthConfig { + registries?: StoredRegistryEntry[]; + // Legacy fields for backwards compatibility + encrypted_token?: encrypted_config; + registry_url?: string; +} + +export interface DecryptedRegistryEntry { + id: string; + scope?: string; + registry_url: string; + token: string; + updated_at: string; +} \ No newline at end of file From 893df207cab5f6308748b0e25d3f4a8357b65109 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 18 Oct 2026 18:49:34 +0530 Subject: [PATCH 44/46] fix: enhance provider registration logic and update channel routing handling --- src/plugins/loader/registry.ts | 21 +++++- .../services/provider-manager.service.ts | 26 +++++++- src/processors/unified/unified.processor.ts | 4 ++ .../plugins/provider-manager.service.test.ts | 66 +++++++++++++++++++ tests/unit/plugins/registry.test.ts | 26 ++++++++ 5 files changed, 139 insertions(+), 4 deletions(-) diff --git a/src/plugins/loader/registry.ts b/src/plugins/loader/registry.ts index 7516584..e3741bd 100644 --- a/src/plugins/loader/registry.ts +++ b/src/plugins/loader/registry.ts @@ -117,9 +117,28 @@ class PluginRegistryClass { */ registerOrReplace(provider: SimpleNSProvider, id: string, priority: number = 0): void { if (this.providers.has(id)) { + const registered = this.providers.get(id); + const channel = registered?.provider.manifest.channel; + const savedConfig = channel ? this.channelConfig.get(channel) : undefined; + const wasDefault = savedConfig?.default === id; + const fallbackIndex = Array.isArray(savedConfig?.fallback) + ? savedConfig.fallback.indexOf(id) + : -1; + this.unregister(id); + this.register(provider, id, priority); + + if (savedConfig) { + if (wasDefault) { + savedConfig.default = id; + } + if (fallbackIndex !== -1 && Array.isArray(savedConfig.fallback) && !savedConfig.fallback.includes(id)) { + savedConfig.fallback.splice(fallbackIndex, 0, id); + } + } + } else { + this.register(provider, id, priority); } - this.register(provider, id, priority); } /** diff --git a/src/plugins/services/provider-manager.service.ts b/src/plugins/services/provider-manager.service.ts index 2640603..18ee4f8 100644 --- a/src/plugins/services/provider-manager.service.ts +++ b/src/plugins/services/provider-manager.service.ts @@ -266,9 +266,25 @@ export class ProviderManagerService { (providerId: string) => providerId !== id ); + let updatedDefault = routing.default_provider_id; + let updatedFallbacks = remainingFallbacks; + if (routing.default_provider_id === id) { - const [nextDefault, ...fallbacks] = remainingFallbacks; + const enabledProviders = await Provider.find({ + channel: routing.channel, + enabled: true, + id: { $ne: id }, + }).lean(); + const enabledProviderIds = new Set(enabledProviders.map((p) => p.id)); + + const nextDefault = remainingFallbacks.find((pid: string) => enabledProviderIds.has(pid)) + || remainingFallbacks[0] + || enabledProviders[0]?.id; + if (nextDefault) { + const fallbacks = remainingFallbacks.filter((pid: string) => pid !== nextDefault); + updatedDefault = nextDefault; + updatedFallbacks = fallbacks; await ChannelRouting.updateOne( { channel: routing.channel }, { @@ -277,6 +293,8 @@ export class ProviderManagerService { } ); } else { + updatedDefault = ''; + updatedFallbacks = []; await ChannelRouting.deleteOne({ channel: routing.channel }); } } else { @@ -287,8 +305,8 @@ export class ProviderManagerService { } PluginRegistry.setChannelConfig(routing.channel, { - default: remainingFallbacks[0] || '', - fallback: remainingFallbacks.slice(1), + default: updatedDefault, + fallback: updatedFallbacks, }); await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { channel: routing.channel, @@ -487,6 +505,7 @@ export class ProviderManagerService { if (!provider_id) return; logger.info(`Sync: Handling remote PROVIDER_UPSERTED for '${provider_id}'...`); + ProviderManagerService.invalidateCredentialCache(provider_id); const providerDoc = await Provider.findOne({ id: provider_id }); if (providerDoc && providerDoc.enabled) { @@ -505,6 +524,7 @@ export class ProviderManagerService { if (!provider_id) return; logger.info(`Sync: Handling remote PROVIDER_DELETED for '${provider_id}'...`); + ProviderManagerService.invalidateCredentialCache(provider_id); PluginRegistry.unregister(provider_id); }); } diff --git a/src/processors/unified/unified.processor.ts b/src/processors/unified/unified.processor.ts index 56cccd6..80f0197 100644 --- a/src/processors/unified/unified.processor.ts +++ b/src/processors/unified/unified.processor.ts @@ -244,6 +244,10 @@ const main = async (): Promise => { await syncActiveConsumers(); }); + PluginSyncService.on('PROVIDER_DELETED', async () => { + await syncActiveConsumers(); + }); + // 4. Load plugins and providers from MongoDB logger.info('Loading providers from MongoDB...'); await loadProvidersFromDatabase({ initialize: true }); diff --git a/tests/unit/plugins/provider-manager.service.test.ts b/tests/unit/plugins/provider-manager.service.test.ts index ef8c56f..dc8c7aa 100644 --- a/tests/unit/plugins/provider-manager.service.test.ts +++ b/tests/unit/plugins/provider-manager.service.test.ts @@ -215,8 +215,15 @@ describe('ProviderManagerService', () => { }, ]), } as any); + vi.spyOn(Provider, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { id: 'smtp-prod', channel: 'email', enabled: true }, + { id: 'backup-prod', channel: 'email', enabled: true }, + ]), + } as any); const updateSpy = vi.spyOn(ChannelRouting, 'updateOne').mockResolvedValue({ acknowledged: true } as any); const deleteRoutingSpy = vi.spyOn(ChannelRouting, 'deleteOne').mockResolvedValue({ deletedCount: 0 } as any); + const setChannelConfigSpy = vi.spyOn(PluginRegistry, 'setChannelConfig'); vi.spyOn(Provider, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); @@ -229,10 +236,61 @@ describe('ProviderManagerService', () => { fallback_provider_ids: ['backup-prod'], } ); + expect(setChannelConfigSpy).toHaveBeenCalledWith('email', { + default: 'smtp-prod', + fallback: ['backup-prod'], + }); expect(deleteRoutingSpy).not.toHaveBeenCalled(); expect(PluginSyncService.publish).toHaveBeenCalledWith('CHANNEL_ROUTING_UPDATED', { channel: 'email', }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_DELETED', { + provider_id: 'gmail-prod', + }); + }); + + it('should retain default provider and only update fallbacks when deleting a fallback provider', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ id: 'smtp-prod' } as any); + vi.spyOn(ChannelRouting, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + channel: 'email', + default_provider_id: 'gmail-prod', + fallback_provider_ids: ['smtp-prod', 'backup-prod'], + }, + ]), + } as any); + vi.spyOn(Provider, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { id: 'gmail-prod', channel: 'email', enabled: true }, + { id: 'backup-prod', channel: 'email', enabled: true }, + ]), + } as any); + const updateSpy = vi.spyOn(ChannelRouting, 'updateOne').mockResolvedValue({ acknowledged: true } as any); + const deleteRoutingSpy = vi.spyOn(ChannelRouting, 'deleteOne').mockResolvedValue({ deletedCount: 0 } as any); + const setChannelConfigSpy = vi.spyOn(PluginRegistry, 'setChannelConfig'); + vi.spyOn(Provider, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); + vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); + + await ProviderManagerService.deleteProvider('smtp-prod'); + + expect(updateSpy).toHaveBeenCalledWith( + { channel: 'email' }, + { + fallback_provider_ids: ['backup-prod'], + } + ); + expect(setChannelConfigSpy).toHaveBeenCalledWith('email', { + default: 'gmail-prod', + fallback: ['backup-prod'], + }); + expect(deleteRoutingSpy).not.toHaveBeenCalled(); + expect(PluginSyncService.publish).toHaveBeenCalledWith('CHANNEL_ROUTING_UPDATED', { + channel: 'email', + }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_DELETED', { + provider_id: 'smtp-prod', + }); }); it('should delete the routing when the provider has no fallback replacement', async () => { @@ -246,13 +304,21 @@ describe('ProviderManagerService', () => { }, ]), } as any); + vi.spyOn(Provider, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([]), + } as any); const deleteRoutingSpy = vi.spyOn(ChannelRouting, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); const deleteSpy = vi.spyOn(Provider, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); const unregisterSpy = vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); + const setChannelConfigSpy = vi.spyOn(PluginRegistry, 'setChannelConfig'); await ProviderManagerService.deleteProvider('mock-unused'); expect(deleteRoutingSpy).toHaveBeenCalledWith({ channel: 'mock' }); + expect(setChannelConfigSpy).toHaveBeenCalledWith('mock', { + default: '', + fallback: [], + }); expect(deleteSpy).toHaveBeenCalledWith({ id: 'mock-unused' }); expect(unregisterSpy).toHaveBeenCalledWith('mock-unused'); expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_DELETED', { diff --git a/tests/unit/plugins/registry.test.ts b/tests/unit/plugins/registry.test.ts index 766e8ea..ff621d8 100644 --- a/tests/unit/plugins/registry.test.ts +++ b/tests/unit/plugins/registry.test.ts @@ -92,6 +92,32 @@ describe('Plugin Registry', () => { }); }); + describe('registerOrReplace', () => { + it('should register a new provider if not present', () => { + const gmail = createMockProvider({ name: 'gmail', channel: 'email' }); + PluginRegistry.registerOrReplace(gmail as any, 'gmail', 1); + + expect(PluginRegistry.has('gmail')).toBe(true); + expect(PluginRegistry.get('gmail')).toBeDefined(); + }); + + it('should replace an existing provider and preserve channelConfig default and fallback', () => { + const gmail1 = createMockProvider({ name: 'gmail-v1', channel: 'email' }); + const gmail2 = createMockProvider({ name: 'gmail-v2', channel: 'email' }); + const smtp = createMockProvider({ name: 'smtp', channel: 'email' }); + + PluginRegistry.register(gmail1 as any, 'gmail', 1); + PluginRegistry.register(smtp as any, 'smtp', 2); + PluginRegistry.setChannelConfig('email', { default: 'gmail', fallback: ['smtp'] }); + + PluginRegistry.registerOrReplace(gmail2 as any, 'gmail', 5); + + expect(PluginRegistry.get('gmail')?.manifest.name).toBe('gmail-v2'); + expect(PluginRegistry.getDefaultProviderId('email')).toBe('gmail'); + expect(PluginRegistry.getFallbackProviderIds('email')).toEqual(['smtp']); + }); + }); + describe('get', () => { it('should return provider by ID', () => { const mockProvider = createMockProvider({ name: 'gmail', channel: 'email' }); From 4f19b5378043a80d671e8db5cd5ab46805b52241 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Sun, 18 Oct 2026 19:10:58 +0530 Subject: [PATCH 45/46] test: add unit tests for YamlMigrator migration logic and provider handling --- src/plugins/bootstrap/yaml-migrator.ts | 182 ++++++++++----- tests/unit/plugins/yaml-migrator.test.ts | 282 +++++++++++++++++++++++ 2 files changed, 408 insertions(+), 56 deletions(-) create mode 100644 tests/unit/plugins/yaml-migrator.test.ts diff --git a/src/plugins/bootstrap/yaml-migrator.ts b/src/plugins/bootstrap/yaml-migrator.ts index 48ff56b..3dccab5 100644 --- a/src/plugins/bootstrap/yaml-migrator.ts +++ b/src/plugins/bootstrap/yaml-migrator.ts @@ -69,73 +69,107 @@ export class YamlMigrator { // 1. Process and install plugins const processedPlugins = new Set(); + const failedPlugins = new Set(); for (const entry of rawConfig.providers) { const packageName = entry.package; - if (!processedPlugins.has(packageName)) { - const targetVersion = entry.version || 'latest'; - const installedPath = join(PLUGINS_NODE_MODULES, packageName); - - // If not already in node_modules, install it - if (!existsSync(installedPath)) { - logger.info(`Migrating plugin: Installing '${packageName}@${targetVersion}'...`); - try { - installNpmPackage(packageName, targetVersion); - } catch (installErr) { - logger.error(`Could not install plugin package '${packageName}':`, installErr); - } - } + if (processedPlugins.has(packageName) || failedPlugins.has(packageName)) { + continue; + } + const targetVersion = entry.version || 'latest'; + const installedPath = join(PLUGINS_NODE_MODULES, packageName); + + // If not already in node_modules, install it + let installFailed = false; + if (!existsSync(installedPath)) { + logger.info(`Migrating plugin: Installing '${packageName}@${targetVersion}'...`); try { - const { manifest, version: authoritativeVersion } = await extractPackageManifest(packageName); - await Plugin.findOneAndUpdate( - { name: packageName }, - { - name: packageName, - version: authoritativeVersion, - status: 'installed', - manifest, - error: undefined, - }, - { upsert: true, new: true } - ); - processedPlugins.add(packageName); - logger.success(`Migrated plugin '${packageName}' (version: ${authoritativeVersion})`); - } catch (manifestErr) { - logger.error(`Failed to extract manifest for '${packageName}':`, manifestErr); + installNpmPackage(packageName, targetVersion); + } catch (installErr) { + logger.error(`Could not install plugin package '${packageName}':`, installErr); + installFailed = true; } } + + if (installFailed) { + failedPlugins.add(packageName); + continue; + } + + try { + const { manifest, version: authoritativeVersion } = await extractPackageManifest(packageName); + if (!manifest?.channel) { + throw new Error(`Package '${packageName}' manifest does not specify a valid channel.`); + } + + await Plugin.findOneAndUpdate( + { name: packageName }, + { + name: packageName, + version: authoritativeVersion, + status: 'installed', + manifest, + error: undefined, + }, + { upsert: true, new: true } + ); + processedPlugins.add(packageName); + logger.success(`Migrated plugin '${packageName}' (version: ${authoritativeVersion})`); + } catch (manifestErr) { + logger.error(`Failed to extract manifest for '${packageName}':`, manifestErr); + failedPlugins.add(packageName); + } } // 2. Process and encrypt provider configurations + const migratedProviderIds = new Set(); + for (const entry of rawConfig.providers) { - const resolvedCreds = resolveCredentials(entry.credentials); - const resolvedOpts = resolveOptionalConfig(entry.optionalConfig); - const mergedOptions = { - ...(entry.options || {}), - ...resolvedOpts, - }; + // Find channel from installed plugin manifest + const pluginDoc = await Plugin.findOne({ name: entry.package, status: 'installed' }); + const channel = pluginDoc?.manifest?.channel; - const encryptedCreds = await encryptCredentials(resolvedCreds); + if (!pluginDoc || !channel) { + logger.error( + `Skipping migration for provider '${entry.id}': Plugin '${entry.package}' is not installed or has no valid channel in manifest.` + ); + continue; + } - // Find channel from installed plugin manifest - const pluginDoc = await Plugin.findOne({ name: entry.package }); - const channel = pluginDoc?.manifest?.channel || 'generic'; - - await Provider.findOneAndUpdate( - { id: entry.id }, - { - id: entry.id, - plugin_name: entry.package, - channel, - enabled: true, - credentials: encryptedCreds, - options: mergedOptions, - }, - { upsert: true, new: true } - ); + try { + const resolvedCreds = resolveCredentials(entry.credentials); + const resolvedOpts = resolveOptionalConfig(entry.optionalConfig); + const mergedOptions = { + ...(entry.options || {}), + ...resolvedOpts, + }; + + const encryptedCreds = await encryptCredentials(resolvedCreds); + + await Provider.findOneAndUpdate( + { id: entry.id }, + { + id: entry.id, + plugin_name: entry.package, + channel, + enabled: true, + credentials: encryptedCreds, + options: mergedOptions, + }, + { upsert: true, new: true } + ); + + migratedProviderIds.add(entry.id); + logger.success(`Migrated provider '${entry.id}' (${channel})`); + } catch (providerErr) { + logger.error(`Failed to migrate provider '${entry.id}':`, providerErr); + } + } - logger.success(`Migrated provider '${entry.id}' (${channel})`); + if (migratedProviderIds.size === 0) { + logger.warn('No providers were successfully migrated from YAML configuration.'); + return false; } // 3. Process channel routing @@ -148,19 +182,55 @@ export class YamlMigrator { fallbackArray = [route.fallback.trim()]; } + // Filter fallback providers to those actually migrated and matching this channel + const validFallbacks: string[] = []; + for (const fId of fallbackArray) { + if (migratedProviderIds.has(fId)) { + validFallbacks.push(fId); + } else { + const exists = await Provider.findOne({ id: fId, channel }); + if (exists) { + validFallbacks.push(fId); + } else { + logger.warn( + `Omitting fallback provider '${fId}' from channel routing '${channel}': provider not migrated or does not exist for this channel.` + ); + } + } + } + + // Validate default provider + let effectiveDefault = (migratedProviderIds.has(route.default) || (await Provider.findOne({ id: route.default, channel }))) + ? route.default + : undefined; + + if (!effectiveDefault) { + if (validFallbacks.length > 0) { + effectiveDefault = validFallbacks.shift()!; + logger.warn( + `Default provider '${route.default}' for channel '${channel}' was not migrated. Promoted fallback '${effectiveDefault}' as default.` + ); + } else { + logger.error( + `Skipping channel routing for '${channel}': default provider '${route.default}' was not migrated and no valid fallbacks are available.` + ); + continue; + } + } + await ChannelRouting.findOneAndUpdate( { channel }, { channel, - default_provider_id: route.default, - fallback_provider_ids: fallbackArray, + default_provider_id: effectiveDefault, + fallback_provider_ids: validFallbacks, partitions: route.partitions || 6, }, { upsert: true, new: true } ); logger.success( - `Migrated channel routing '${channel}': default=${route.default}, fallbacks=[${fallbackArray.join(', ')}]` + `Migrated channel routing '${channel}': default=${effectiveDefault}, fallbacks=[${validFallbacks.join(', ')}]` ); } } diff --git a/tests/unit/plugins/yaml-migrator.test.ts b/tests/unit/plugins/yaml-migrator.test.ts new file mode 100644 index 0000000..4b7dc7b --- /dev/null +++ b/tests/unit/plugins/yaml-migrator.test.ts @@ -0,0 +1,282 @@ +/** + * Unit Tests for YamlMigrator + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import * as fs from 'fs'; +import { YamlMigrator } from '../../../src/plugins/bootstrap/yaml-migrator.js'; +import Plugin from '../../../src/database/models/plugin.models.js'; +import Provider from '../../../src/database/models/provider.models.js'; +import ChannelRouting from '../../../src/database/models/channel-routing.models.js'; +import * as pluginFs from '../../../src/plugins/loader/plugin-fs.js'; +import * as keypairManager from '../../../src/plugins/crypto/keypair-manager.js'; +import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; + +vi.mock('fs', async () => { + const actual = await vi.importActual('fs'); + return { + ...actual, + existsSync: vi.fn(), + readFileSync: vi.fn(), + }; +}); + +describe('YamlMigrator Unit Tests', () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.spyOn(PluginSyncService, 'publish').mockResolvedValue(undefined); + vi.spyOn(keypairManager, 'encryptCredentials').mockResolvedValue({ + encrypted_data: 'enc-data', + iv: 'enc-iv', + auth_tag: 'enc-tag', + encrypted_dek: 'enc-dek', + }); + }); + + it('should skip migration if providers already exist in MongoDB', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(2 as any); + const findConfigSpy = vi.spyOn(pluginFs, 'findConfigFile'); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(false); + expect(findConfigSpy).not.toHaveBeenCalled(); + }); + + it('should return false if no config file is found', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue(null); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(false); + }); + + it('should return false if config has no providers', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue('./simplens.config.yaml'); + vi.mocked(fs.readFileSync).mockReturnValue('channels: {}'); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(false); + }); + + it('should skip provider creation and not create generic provider if npm install fails', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue('./simplens.config.yaml'); + vi.mocked(fs.existsSync).mockReturnValue(false); + vi.mocked(fs.readFileSync).mockReturnValue(` +providers: + - id: broken-provider + package: '@simplens/broken-pkg' + credentials: + apiKey: test-key +`); + + vi.spyOn(pluginFs, 'installNpmPackage').mockImplementation(() => { + throw new Error('NPM install 404 Not Found'); + }); + const extractManifestSpy = vi.spyOn(pluginFs, 'extractPackageManifest'); + const providerUpsertSpy = vi.spyOn(Provider, 'findOneAndUpdate'); + const pluginUpsertSpy = vi.spyOn(Plugin, 'findOneAndUpdate'); + vi.spyOn(Plugin, 'findOne').mockResolvedValue(null); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + // Migration returns false because 0 providers were migrated + expect(result).toBe(false); + // Manifest extraction shouldn't be called if install failed + expect(extractManifestSpy).not.toHaveBeenCalled(); + // No Plugin record created + expect(pluginUpsertSpy).not.toHaveBeenCalled(); + // CRUCIAL: No Provider record created, generic provider avoided + expect(providerUpsertSpy).not.toHaveBeenCalled(); + }); + + it('should skip provider creation if manifest extraction fails', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue('./simplens.config.yaml'); + vi.mocked(fs.existsSync).mockReturnValue(false); + vi.mocked(fs.readFileSync).mockReturnValue(` +providers: + - id: broken-manifest-provider + package: '@simplens/corrupted-pkg' + credentials: + apiKey: test-key +`); + + vi.spyOn(pluginFs, 'installNpmPackage').mockReturnValue(undefined); + vi.spyOn(pluginFs, 'extractPackageManifest').mockRejectedValue(new Error('Invalid package manifest')); + const providerUpsertSpy = vi.spyOn(Provider, 'findOneAndUpdate'); + const pluginUpsertSpy = vi.spyOn(Plugin, 'findOneAndUpdate'); + vi.spyOn(Plugin, 'findOne').mockResolvedValue(null); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(false); + expect(pluginUpsertSpy).not.toHaveBeenCalled(); + expect(providerUpsertSpy).not.toHaveBeenCalled(); + }); + + it('should migrate valid provider and omit failed provider, promoting fallback if default failed', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue('./simplens.config.yaml'); + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.readFileSync).mockReturnValue(` +providers: + - id: default-email + package: '@simplens/broken-email' + credentials: + key: val + - id: fallback-email + package: '@simplens/good-email' + credentials: + key: val +channels: + email: + default: default-email + fallback: + - fallback-email +`); + + // First package fails manifest, second succeeds + vi.spyOn(pluginFs, 'extractPackageManifest').mockImplementation(async (pkg: string) => { + if (pkg === '@simplens/broken-email') { + throw new Error('Broken manifest'); + } + return { + manifest: { + name: '@simplens/good-email', + displayName: 'Good Email Provider', + version: '1.0.0', + channel: 'email', + description: 'Good email provider', + requiredCredentials: ['key'], + }, + version: '1.0.0', + }; + }); + + const pluginUpsertSpy = vi.spyOn(Plugin, 'findOneAndUpdate').mockResolvedValue({} as any); + const providerUpsertSpy = vi.spyOn(Provider, 'findOneAndUpdate').mockResolvedValue({} as any); + const routingUpsertSpy = vi.spyOn(ChannelRouting, 'findOneAndUpdate').mockResolvedValue({} as any); + + vi.spyOn(Plugin, 'findOne').mockImplementation(async (query: any) => { + if (query.name === '@simplens/good-email') { + return { + name: '@simplens/good-email', + status: 'installed', + manifest: { channel: 'email' }, + } as any; + } + return null; + }); + + vi.spyOn(Provider, 'findOne').mockImplementation(async (query: any) => { + if (query.id === 'fallback-email' && query.channel === 'email') { + return { id: 'fallback-email', channel: 'email' } as any; + } + return null; + }); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(true); + + // Only good-email plugin is saved + expect(pluginUpsertSpy).toHaveBeenCalledTimes(1); + expect(pluginUpsertSpy).toHaveBeenCalledWith( + { name: '@simplens/good-email' }, + expect.objectContaining({ name: '@simplens/good-email', status: 'installed' }), + expect.any(Object) + ); + + // Only fallback-email provider is created (with channel 'email') + expect(providerUpsertSpy).toHaveBeenCalledTimes(1); + expect(providerUpsertSpy).toHaveBeenCalledWith( + { id: 'fallback-email' }, + expect.objectContaining({ + id: 'fallback-email', + plugin_name: '@simplens/good-email', + channel: 'email', + enabled: true, + }), + expect.any(Object) + ); + + // Channel routing promoted fallback-email to default since default-email failed + expect(routingUpsertSpy).toHaveBeenCalledWith( + { channel: 'email' }, + expect.objectContaining({ + channel: 'email', + default_provider_id: 'fallback-email', + fallback_provider_ids: [], + }), + expect.any(Object) + ); + }); + + it('should migrate all providers and channel routings successfully when all succeed', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue('./simplens.config.yaml'); + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.readFileSync).mockReturnValue(` +providers: + - id: resend-provider + package: '@simplens/resend' + credentials: + apiKey: resend-123 +channels: + email: + default: resend-provider +`); + + vi.spyOn(pluginFs, 'extractPackageManifest').mockResolvedValue({ + manifest: { + name: '@simplens/resend', + displayName: 'Resend Provider', + version: '1.2.0', + channel: 'email', + description: 'Resend email provider', + requiredCredentials: ['apiKey'], + }, + version: '1.2.0', + }); + + vi.spyOn(Plugin, 'findOneAndUpdate').mockResolvedValue({} as any); + vi.spyOn(Provider, 'findOneAndUpdate').mockResolvedValue({} as any); + vi.spyOn(ChannelRouting, 'findOneAndUpdate').mockResolvedValue({} as any); + + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ + name: '@simplens/resend', + status: 'installed', + manifest: { channel: 'email' }, + } as any); + + const publishSpy = vi.spyOn(PluginSyncService, 'publish'); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(true); + expect(Provider.findOneAndUpdate).toHaveBeenCalledWith( + { id: 'resend-provider' }, + expect.objectContaining({ + id: 'resend-provider', + plugin_name: '@simplens/resend', + channel: 'email', + enabled: true, + }), + expect.any(Object) + ); + expect(ChannelRouting.findOneAndUpdate).toHaveBeenCalledWith( + { channel: 'email' }, + expect.objectContaining({ + channel: 'email', + default_provider_id: 'resend-provider', + }), + expect.any(Object) + ); + expect(publishSpy).toHaveBeenCalledWith('SYSTEM_RELOAD', {}); + }); +}); From 0fafc8793f82353f2d86c566f14a2af765e41669 Mon Sep 17 00:00:00 2001 From: Adhish-Krishna Date: Mon, 19 Oct 2026 20:39:16 +0530 Subject: [PATCH 46/46] fix: update environment configurations for production readiness and improve service definitions --- .env.example | 20 ++--- dashboard/.env.example | 3 - docker-compose/docker-compose.dev.yaml | 105 +++------------------- docker-compose/docker-compose.local.yaml | 101 +++------------------ docker-compose/docker-compose.master.yaml | 104 +++------------------ packages/onboard/src/env-config.ts | 3 - 6 files changed, 39 insertions(+), 297 deletions(-) diff --git a/.env.example b/.env.example index 9a4a15d..5b326f1 100644 --- a/.env.example +++ b/.env.example @@ -12,7 +12,7 @@ # ============================================ # API SERVER # ============================================ -NODE_ENV=development # Set to 'production', 'development', or 'test' +NODE_ENV=production # Set to 'production', 'development', or 'test' PORT=3000 # API server port JWT_SECRET= # REQUIRED: Secret key for JWT admin tokens (min 32 chars, generate with: openssl rand -base64 32) @@ -20,31 +20,27 @@ JWT_SECRET= # REQUIRED: Secret key for JWT admin tokens ( # DATABASE # ============================================ # MongoDB connection URI (configured with replicaSet for change streams/transactions) -MONGO_URI=mongodb://127.0.0.1:27017/notification_service?replicaSet=rs0 - +MONGO_URI=mongodb://mongo:27017/simplens?replicaSet=rs0 # ============================================ # KAFKA # ============================================ -BROKERS=localhost:9092 # Comma-separated Kafka broker addresses +BROKERS=kafka:9093 # Comma-separated Kafka broker addresses # ============================================ # REDIS # ============================================ -REDIS_URL=redis://localhost:6379 # Redis connection URL (used for pub/sub sync & locks) +REDIS_URL=redis://redis:6379 # Redis connection URL (used for pub/sub sync & locks) # ============================================ # LOGGING (Grafana/Loki) # ============================================ -LOKI_URL=http://localhost:3100 # Optional Loki endpoint for centralized logging +LOKI_URL=http://loki:3100 # Optional Loki endpoint for centralized logging LOG_LEVEL=info # Initial bootstrap log level (debug, info, warn, error) LOG_TO_FILE=true # Enable file transport logging to /logs directory # ============================================ # ADMIN DASHBOARD # ============================================ -AUTH_TRUST_HOST=true # Trust host header for authentication cookies -API_BASE_URL=http://localhost:3000 # Core Notification service API URL -WEBHOOK_HOST=dashboard # Webhook callback host -WEBHOOK_PORT=3002 # Dashboard webhook port -DASHBOARD_PORT=3002 # Next.js Dashboard port -BASE_PATH=/dashboard # Optional subpath (e.g. /dashboard) or empty for root \ No newline at end of file +API_BASE_URL=http://api:3000/sns # Core Notification service API URL +DASHBOARD_PORT=3002 # Next.js Dashboard port +BASE_PATH=/sns # Optional subpath (e.g. /dashboard) or empty for root \ No newline at end of file diff --git a/dashboard/.env.example b/dashboard/.env.example index 4fb8724..9c73113 100644 --- a/dashboard/.env.example +++ b/dashboard/.env.example @@ -1,7 +1,4 @@ NODE_ENV= # set as production, development, test etc.. used for setting the secure propery for cookie -MONGO_URI= # mongodb connection uri PORT=3002 # port number API_BASE_URL= # notification service base url (e.g. http://localhost:3000) -WEBHOOK_HOST= # webhook host -WEBHOOK_PORT= # webhook port BASE_PATH= # Base path for Next.js app (e.g., /dashboard). Leave empty for root path. diff --git a/docker-compose/docker-compose.dev.yaml b/docker-compose/docker-compose.dev.yaml index b007df1..7701095 100644 --- a/docker-compose/docker-compose.dev.yaml +++ b/docker-compose/docker-compose.dev.yaml @@ -1,178 +1,95 @@ # ============================================================================== -# SimpleNS Development Environment (GHCR 'development' branch images) +# SimpleNS Dev Environment # Connects to shared 'simplens-infra' network and 'simplens_dev' database. # ============================================================================== name: simplens-dev - services: - # ============================================ - # 1. API Server - # ============================================ api: image: ghcr.io/simplenotificationsystem/simplens-core:development container_name: simplens-api-dev ports: - - "3000:3000" + - 3000:3000 env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-development} - PORT: ${PORT:-3000} - MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - JWT_SECRET: ${JWT_SECRET} - SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} volumes: - logs-data-dev:/app/logs command: [ "node", "dist/api/server.js" ] + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped - healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000${BASE_PATH:-}/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] - interval: 15s - timeout: 5s - retries: 3 - start_period: 10s labels: simplens.environment: "dev" - # ============================================ - # 2. Outbox Worker - # ============================================ worker: image: ghcr.io/simplenotificationsystem/simplens-core:development container_name: simplens-worker-dev env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-development} - MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-dev:/app/logs command: [ "node", "dist/workers/worker.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "dev" - # ============================================ - # 3. Unified Notification Processor - # ============================================ + notification_processor: image: ghcr.io/simplenotificationsystem/simplens-core:development container_name: simplens-notification-processor-dev env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-development} - MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - PROCESSOR_CHANNEL: ${PROCESSOR_CHANNEL:-all} - SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-dev:/app/logs command: [ "node", "dist/processors/unified/unified.processor.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "dev" - # ============================================ - # 4. Delayed Processor (Scheduled Notifications) - # ============================================ delayed_processor: image: ghcr.io/simplenotificationsystem/simplens-core:development container_name: simplens-delayed-processor-dev env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-development} - MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-dev:/app/logs command: [ "node", "dist/processors/delayed/delayed.processor.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "dev" - # ============================================ - # 5. Recovery Service (Stuck Notifications & Audits) - # ============================================ recovery: image: ghcr.io/simplenotificationsystem/simplens-core:development container_name: simplens-recovery-dev env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-development} - MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-dev:/app/logs command: [ "node", "dist/workers/recovery/recovery.service.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "dev" - # ============================================ - # 6. Admin Dashboard (Next.js) - # ============================================ dashboard: image: ghcr.io/simplenotificationsystem/simplens-dashboard:development - container_name: simplens-dashboard-dev ports: - - "3002:3002" + - 3002:3002 + container_name: simplens-dashboard-dev env_file: - ../.env environment: - NODE_ENV: production PORT: ${DASHBOARD_PORT:-3002} - API_BASE_URL: http://api:{PORT:-3000}{BASE_PATH:-} - BASE_PATH: ${BASE_PATH:-} - MONGO_URI: mongodb://mongo:27017/simplens_dev?replicaSet=rs0 - AUTH_TRUST_HOST: "true" - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "dev" diff --git a/docker-compose/docker-compose.local.yaml b/docker-compose/docker-compose.local.yaml index c12f065..0a11abe 100644 --- a/docker-compose/docker-compose.local.yaml +++ b/docker-compose/docker-compose.local.yaml @@ -1,14 +1,10 @@ # ============================================================================== -# SimpleNS Local Environment (Built from Local Source Code) +# SimpleNS Local Environment # Connects to shared 'simplens-infra' network and 'simplens_local' database. # ============================================================================== name: simplens-local - services: - # ============================================ - # 1. API Server - # ============================================ api: build: context: .. @@ -16,166 +12,89 @@ services: image: simplens-core:local container_name: simplens-api-local ports: - - "3000:3000" + - 3000:3000 env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-development} - PORT: ${PORT:-3000} - MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - JWT_SECRET: ${JWT_SECRET} volumes: - logs-data-local:/app/logs command: [ "node", "dist/api/server.js" ] + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped - healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000${BASE_PATH:-}/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] - interval: 15s - timeout: 5s - retries: 3 - start_period: 10s labels: simplens.environment: "local" - # ============================================ - # 2. Outbox Worker - # ============================================ worker: image: simplens-core:local container_name: simplens-worker-local env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-development} - MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-local:/app/logs command: [ "node", "dist/workers/worker.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "local" - # ============================================ - # 3. Unified Notification Processor - # ============================================ notification_processor: image: simplens-core:local container_name: simplens-notification-processor-local env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-development} - MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-local:/app/logs command: [ "node", "dist/processors/unified/unified.processor.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "local" - # ============================================ - # 4. Delayed Processor (Scheduled Notifications) - # ============================================ delayed_processor: image: simplens-core:local container_name: simplens-delayed-processor-local env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-development} - MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-local:/app/logs command: [ "node", "dist/processors/delayed/delayed.processor.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "local" - # ============================================ - # 5. Recovery Service (Stuck Notifications & Audits) - # ============================================ recovery: image: simplens-core:local container_name: simplens-recovery-local env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-development} - MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-local:/app/logs command: [ "node", "dist/workers/recovery/recovery.service.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "local" - # ============================================ - # 6. Admin Dashboard (Next.js) - # ============================================ dashboard: build: context: ../dashboard dockerfile: Dockerfile image: simplens-dashboard:local - container_name: simplens-dashboard-local ports: - - "3002:3002" + - 3002:3002 + container_name: simplens-dashboard-local env_file: - ../.env environment: - NODE_ENV: ${NODE_ENV:-development} PORT: ${DASHBOARD_PORT:-3002} - API_BASE_URL: http://api:{PORT:-3000}{BASE_PATH:-} - BASE_PATH: ${BASE_PATH:-} - MONGO_URI: mongodb://mongo:27017/simplens_local?replicaSet=rs0 - AUTH_TRUST_HOST: "true" - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "local" diff --git a/docker-compose/docker-compose.master.yaml b/docker-compose/docker-compose.master.yaml index b5a2fb6..4faca4f 100644 --- a/docker-compose/docker-compose.master.yaml +++ b/docker-compose/docker-compose.master.yaml @@ -1,178 +1,94 @@ # ============================================================================== -# SimpleNS Master Environment (GHCR 'latest' / master branch images) +# SimpleNS Master Environment # Connects to shared 'simplens-infra' network and 'simplens_master' database. # ============================================================================== name: simplens-master - services: - # ============================================ - # 1. API Server - # ============================================ api: image: ghcr.io/simplenotificationsystem/simplens-core:latest container_name: simplens-api-master ports: - - "3000:3000" + - 3000:3000 env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-production} - PORT: ${PORT:-3000} - MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - JWT_SECRET: ${JWT_SECRET} - SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} volumes: - logs-data-master:/app/logs command: [ "node", "dist/api/server.js" ] + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped - healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000${BASE_PATH:-}/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] - interval: 15s - timeout: 5s - retries: 3 - start_period: 10s labels: simplens.environment: "master" - # ============================================ - # 2. Outbox Worker - # ============================================ worker: image: ghcr.io/simplenotificationsystem/simplens-core:latest container_name: simplens-worker-master env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-master:/app/logs command: [ "node", "dist/workers/worker.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "master" - # ============================================ - # 3. Unified Notification Processor - # ============================================ notification_processor: image: ghcr.io/simplenotificationsystem/simplens-core:latest container_name: simplens-notification-processor-master env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - PROCESSOR_CHANNEL: ${PROCESSOR_CHANNEL:-all} - SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-master:/app/logs command: [ "node", "dist/processors/unified/unified.processor.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "master" - # ============================================ - # 4. Delayed Processor (Scheduled Notifications) - # ============================================ delayed_processor: image: ghcr.io/simplenotificationsystem/simplens-core:latest container_name: simplens-delayed-processor-master env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-master:/app/logs command: [ "node", "dist/processors/delayed/delayed.processor.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "master" - # ============================================ - # 5. Recovery Service (Stuck Notifications & Audits) - # ============================================ recovery: image: ghcr.io/simplenotificationsystem/simplens-core:latest container_name: simplens-recovery-master env_file: - ../.env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - logs-data-master:/app/logs command: [ "node", "dist/workers/recovery/recovery.service.js" ] - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "master" - # ============================================ - # 6. Admin Dashboard (Next.js) - # ============================================ dashboard: image: ghcr.io/simplenotificationsystem/simplens-dashboard:latest - container_name: simplens-dashboard-master ports: - - "3002:3002" + - 3002:3002 + container_name: simplens-dashboard-master env_file: - ../.env environment: - NODE_ENV: production PORT: ${DASHBOARD_PORT:-3002} - API_BASE_URL: http://api:{PORT:-3000}{BASE_PATH:-} - BASE_PATH: ${BASE_PATH:-} - MONGO_URI: mongodb://mongo:27017/simplens_master?replicaSet=rs0 - AUTH_TRUST_HOST: "true" - depends_on: - api: - condition: service_healthy + restart: unless-stopped networks: - simplens-infra - restart: unless-stopped labels: simplens.environment: "master" diff --git a/packages/onboard/src/env-config.ts b/packages/onboard/src/env-config.ts index be83465..c281294 100644 --- a/packages/onboard/src/env-config.ts +++ b/packages/onboard/src/env-config.ts @@ -164,10 +164,7 @@ VERSION=latest # ============================================ # ADMIN DASHBOARD # ============================================ -AUTH_TRUST_HOST=true API_BASE_URL=http://api:3000 -WEBHOOK_HOST=dashboard -WEBHOOK_PORT=3002 DASHBOARD_PORT=3002 BASE_PATH= `;