diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 62fd3e5..069928c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -19,7 +19,7 @@ } }, "postCreateCommand": "npm install && cd dashboard && npm install", - "postStartCommand": "docker compose -f docker-compose.infra.yaml up -d", + "postStartCommand": "docker compose -f ./docker-compose/docker-compose.infra.yaml up -d", "remoteEnv": { "NODE_ENV": "development", "INFRA_HOST": "localhost" diff --git a/.env.example b/.env.example index 85f85e4..5b326f1 100644 --- a/.env.example +++ b/.env.example @@ -1,122 +1,46 @@ -# ============================================ -# INFRASTRUCTURE HOST CONFIGURATION -# ============================================ -# INFRA_HOST: Used by docker-compose.infra.yaml for Kafka & MongoDB configuration -# - Same system: Leave unset (defaults to host.docker.internal) -# - Separate systems: Set to the infrastructure machine's IP/hostname -# ============================================ -# INFRA_HOST=192.168.1.100 # Uncomment and set when running infra on a separate machine +# ============================================================================== +# SimpleNS Essential Environment Configuration +# ============================================================================== +# NOTE: All operational settings (worker batch sizes, poll intervals, retries, +# deduplication TTLs, recovery thresholds, and log levels) are now managed +# dynamically via MongoDB and the Admin Dashboard with live zero-downtime hot-reloading! +# +# Administrator credentials are configured interactively on first launch via +# the Dashboard Setup UI (/setup) and stored securely in MongoDB. +# ============================================================================== -# ============================================ -# CONNECTION URLS (for application services in docker-compose.yaml) -# ============================================ -# Same system: Use host.docker.internal as the host -# Separate systems: Use the infrastructure machine's IP/hostname -# ============================================ - -NODE_ENV= # Set to either production or development # ============================================ # API SERVER # ============================================ -NS_API_KEY= # Generate via: openssl rand -base64 32 +NODE_ENV=production # Set to 'production', 'development', or 'test' PORT=3000 # API server port -MAX_BATCH_REQ_LIMIT=1000 # Max batch request size +JWT_SECRET= # REQUIRED: Secret key for JWT admin tokens (min 32 chars, generate with: openssl rand -base64 32) # ============================================ # DATABASE # ============================================ -MONGO_URI= # MongoDB connection string (If you are using self hosted instance, it should be configured with replicaSet: mongodb://host:port/simplens?replicaSet=rs0) - +# MongoDB connection URI (configured with replicaSet for change streams/transactions) +MONGO_URI=mongodb://mongo:27017/simplens?replicaSet=rs0 # ============================================ # KAFKA # ============================================ -BROKERS= # Kafka brokers list (host:port if self-hosted) - -# Kafka Topic Partitions (Core Topics) -DELAYED_PARTITION=1 # Partitions for delayed_notification topic -NOTIFICATION_STATUS_PARTITION=1 # Partitions for notification_status topic - +BROKERS=kafka:9093 # Comma-separated Kafka broker addresses # ============================================ # REDIS # ============================================ -REDIS_URL= # Redis connection URL (redis://host:port if self-hosted) - -# ============================================ -# PLUGIN SYSTEM -# ============================================ -SIMPLENS_CONFIG_PATH=./simplens.config.yaml # Path to plugin configuration file -PROCESSOR_CHANNEL=all # 'all' for all channels, or specific: 'email', 'sms' - - -# ============================================ -# BACKGROUND WORKER -# ============================================ -OUTBOX_POLL_INTERVAL_MS=5000 # Poll every 5 seconds -OUTBOX_CLEANUP_INTERVAL_MS=60000 # Cleanup every 60 seconds -OUTBOX_BATCH_SIZE=100 # Process 100 entries per poll -OUTBOX_RETENTION_MS=300000 # Keep published entries for 5 minutes -OUTBOX_CLAIM_TIMEOUT_MS=30000 # Claim timeout for crashed workers - -# ============================================ -# RETRY & IDEMPOTENCY -# ============================================ -IDEMPOTENCY_TTL_SECONDS=86400 # Keep idempotency keys for 24 hours -MAX_RETRY_COUNT=5 # Max retries before marking as failed -PROCESSING_TTL_SECONDS=120 # Processing lock held for 2 minutes - -# ============================================ -# DELAYED NOTIFICATIONS -# ============================================ -DELAYED_POLL_INTERVAL_MS=1000 # Poll Redis every 1 second -DELAYED_BATCH_SIZE=10 # Fetch 10 due events per poll -MAX_POLLER_RETRIES=3 # Max retries before DLQ - -# ============================================ -# RECOVERY SERVICE -# ============================================ -RECOVERY_POLL_INTERVAL_MS=60000 # Run recovery every 1 minute -PROCESSING_STUCK_THRESHOLD_MS=300000 # Mark as stuck after 5 minutes -PENDING_STUCK_THRESHOLD_MS=300000 # Mark as orphaned after 5 minutes -RECOVERY_BATCH_SIZE=50 # Process 50 stuck notifications per poll -RECOVERY_CLAIM_TIMEOUT_MS=60000 # Claim timeout for recovery cron worker - -# ============================================ -# CLEANUP (runs as part of recovery) -# ============================================ -CLEANUP_RESOLVED_ALERTS_RETENTION_MS=86400000 # Keep resolved alerts for 24 hours -CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS=86400000 # Keep processed status outbox for 24 hours +REDIS_URL=redis://redis:6379 # Redis connection URL (used for pub/sub sync & locks) # ============================================ # LOGGING (Grafana/Loki) # ============================================ -LOKI_URL= # Loki URL for log aggregation (http://host:port if self-hosted) -LOG_LEVEL=info # Log level: debug, info, warn, error -LOG_TO_FILE=true # Enable file logging +LOKI_URL=http://loki:3100 # Optional Loki endpoint for centralized logging +LOG_LEVEL=info # Initial bootstrap log level (debug, info, warn, error) +LOG_TO_FILE=true # Enable file transport logging to /logs directory # ============================================ # ADMIN DASHBOARD # ============================================ -AUTH_SECRET= # Generate via: openssl rand -base64 32 -ADMIN_USERNAME=admin # Dashboard admin username -ADMIN_PASSWORD= # Dashboard admin password -AUTH_TRUST_HOST=true # Trust host for auth -API_BASE_URL= # Notification service API URL -WEBHOOK_HOST= # Webhook host for callbacks -WEBHOOK_PORT=3002 # Dashboard webhook port -DASHBOARD_PORT=3002 # Dashboard port -BASE_PATH= # Configurable BASE_PATH to render the admin dashboard -HTTPS_COOKIE= # Controls the storage of session cookies in http and https mode. (If set false, then the cookies can be stored in http connection) - - -# ============================================ -# PLUGIN SPECIFIC ENV VARIABLES (Partition variables + Plugin credentials) -# ============================================ - -# Channel-Specific Partitions (Dynamic - based on simplens.config.yaml channels) -# Topic: {CHANNEL}_notification, Env: {CHANNEL}_PARTITION -MOCK_PARTITION=1 # Partitions for mock_notification topic - -# PLUGIN CREDENTIALS (@simplens/nodemailer-gmail) (You need to install this plugin separately. Not included with simplens-core) -EMAIL_USER= # Gmail address -EMAIL_PASS= # Gmail app password (not regular password!) \ No newline at end of file +API_BASE_URL=http://api:3000/sns # Core Notification service API URL +DASHBOARD_PORT=3002 # Next.js Dashboard port +BASE_PATH=/sns # Optional subpath (e.g. /dashboard) or empty for root \ No newline at end of file diff --git a/.env.test b/.env.test index 9ea1e7b..0b90e65 100644 --- a/.env.test +++ b/.env.test @@ -3,6 +3,7 @@ NODE_ENV=test PORT=3099 +JWT_SECRET=test-jwt-secret-key-for-testing-purposes-only # Test Database - uses in-memory MongoDB via mongodb-memory-server # The MONGO_URI below is a placeholder; tests use MongoMemoryServer @@ -14,9 +15,6 @@ REDIS_URL=redis://localhost:6380 # Test Kafka - fully mocked in tests BROKERS=localhost:9093 -# API Authentication -NS_API_KEY=test-api-key-for-testing - # Limits MAX_BATCH_REQ_LIMIT=1000 DELAYED_PARTITION=1 diff --git a/AGENTS.md b/AGENTS.md index d688c65..d912bc4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,9 @@ Use TypeScript with strict typing and existing `@src/*` path aliases. Match curr Plan changes before writing code. Follow low-level design principles: single responsibility, clear interfaces, useful dependency inversion, and explicit error handling. Do not duplicate logic; extract shared behavior into focused utilities, services, or test helpers. Always use `axios` instead of the plain `fetch` API in this project. +## Type & Schema Centralisation +All TypeScript interfaces and types that live inside `src/` **must** be declared in `src/types/types.ts`. All Zod (or other validation library) schemas must be declared in `src/types/schemas.ts`. Individual source files must **not** define their own local interfaces, types, or schemas — import everything from `@src/types/types` or `@src/types/schemas` instead. This rule applies to every layer: API controllers, services, plugins, workers, processors, utilities, and database modules. + ## Testing Guidelines Vitest is the test runner. Integration tests also use `supertest`, `mongodb-memory-server`, and Redis mocks. Name test files with `.test.ts`, for example `tests/unit/plugins/loader.test.ts`. @@ -31,5 +34,11 @@ Use short Conventional Commit style messages such as `feat: add email retry poli PRs should include a summary, linked issue when applicable, test evidence, and screenshots or GIFs for `dashboard/` UI changes. Husky `pre-push` runs both lint commands. +## Dynamic Configuration & Dashboard Settings +Whenever any configuration field is added, modified, or removed in the dynamic operational configuration (`src/types/schemas.ts` in `operationalSettingsSchema`, `src/config/env.config.ts`, or `src/config/dynamic-config.service.ts`), you **must** also update the dashboard: +1. Update the `OperationalSettings` interface in `dashboard/lib/types.ts`. +2. Update the corresponding card and input fields in `dashboard/app/settings/page.tsx` so administrators can view and tune the parameter live with appropriate labels, inputs, and validation hints. + ## Security & Configuration Tips -Do not commit secrets. Use `.env.example` as the template for `.env`, and update it when adding configuration. For local full-stack work, prefer `docker-compose.dev.yaml`; for core-only work, start infrastructure with `docker compose -f docker-compose.infra.yaml up -d`. +Do not commit secrets. Use `.env.example` as the template for `.env`, and update it when adding configuration. For local full-stack work, use `./docker-compose/scripts/switch-env.ps1 -Env local -Rebuild` (or `docker compose -f docker-compose/docker-compose.local.yaml up -d`); for shared infrastructure, use `docker compose -f docker-compose/docker-compose.infra.yaml up -d` (or `./docker-compose/scripts/switch-env.ps1 -Env infra-only`). + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3bce98e..d904281 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,7 @@ This repository contains: Recommended development paths: - Use `.devcontainer` (preferred for easy dev setup without any clutter). -- Use `docker-compose.dev.yaml` for the full stack. +- Use `docker-compose/` for full-stack Docker development with the environment switcher. - Run app code locally only if MongoDB, Redis, and Kafka are available. - Create the `simplens.config.yaml` file for any local development option below. - Refer the [Plugin Configuration Guide](https://www.simplens.in/docs/core/self-hosting#plugin-configuration) for manually creating `simplens.config.yaml` configuration file. @@ -40,18 +40,21 @@ Recommended development paths: 2. Let post-create scripts install dependencies automatically. 3. Start developing with local source mounted in the container. -### Option B: Docker Compose (`docker-compose.dev.yaml`) +### Option B: Docker Compose (`docker-compose/`) -Use when you want the full stack via Docker: +Use when you want the full stack via Docker with the environment switcher: ```bash -docker compose -f docker-compose.dev.yaml build -docker compose -f docker-compose.dev.yaml up -d +# Windows +./docker-compose/scripts/switch-env.ps1 -Env local -Rebuild + +# Linux/macOS +./docker-compose/scripts/switch-env.sh local --rebuild ``` Note: -- Code changes are not always reflected automatically in containerized runtime images. +- Code changes are rebuilt into the local image when passing `-Rebuild` or `--rebuild`. - Rebuild/restart relevant services after source changes when needed. ### Option C: Run Core Locally + Infra Services @@ -75,12 +78,12 @@ cp .env.example .env Copy-Item .env.example .env ``` -3. Start required infrastructure (MongoDB, Kafka, Redis) via Docker: +3. Start required infrastructure (MongoDB, Kafka, Redis, Loki, Grafana) via Docker: -- Set the env variable `INFRA_HOST=localhost` before starting the services in `docker-compose.infra.yaml` +- Set the env variable `INFRA_HOST=localhost` before starting the services in `docker-compose/docker-compose.infra.yaml` ```bash -docker compose -f docker-compose.infra.yaml up -d +docker compose -f docker-compose/docker-compose.infra.yaml up -d ``` ## Development Commands diff --git a/README.md b/README.md index 8de8470..1981999 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ npx -y @simlpens/mcp --stdio ``` *(Supports both `streamable-http` and `stdio` transports)* +Read the [documentation here](https://www.simplens.in/docs/mcp/getting-started) to get started with SimpleNS MCP + --- ## Architecture @@ -110,26 +112,11 @@ SimpleNS is built around a strictly **decoupled, API-first model**: ### Plugin System -SimpleNS Core handles **orchestration**; plugins handle **delivery**. - -Plugins are automatically installed at runtime based on your `simplens.config.yaml` configuration. Use the **Config Generator CLI** to create or update your config: - -```bash -# Generate config for a plugin -npx @simplens/config-gen generate @simplens/mock - -# Generate config for multiple plugins -npx @simplens/config-gen gen @simplens/nodemailer-gmail @simplens/twilio-sms +SimpleNS Core handles **orchestration**, plugins handle **delivery**. -# Add a plugin to existing config -npx @simplens/config-gen gen @simplens/nodemailer-gmail -c simplens.config.yaml +Plugins can be installed directly from the dashboard, via API and MCP server -# List available official plugins -npx @simplens/config-gen list --offical -#List available community plugins -npx @simplens/config-gen list --community -``` #### Building Custom Plugins @@ -207,7 +194,7 @@ Expected response: **Using the Dashboard (recommended):** 1. Open [http://localhost:3002](http://localhost:3002) -2. Login with your configured credentials (default: `admin` / ``) +2. At startup, it lets you to create an admin account, install official and community plugins, setup their corresponding providers and credentials. Then go to dashboard home and create an API key for the external services to access the SimpleNS API server. 3. Navigate to **Send**, select a channel/provider, fill in the fields, and click **Send Notification** **Using cURL:** @@ -215,7 +202,7 @@ Expected response: ```bash curl -X POST http://localhost:3000/api/notification \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer " \ + -H "Authorization: Bearer " \ -d '{ "request_id": "550e8400-e29b-41d4-a716-446655440000", "client_id": "55283667-1f58-467d-86a1-47f7f0e059f2", @@ -246,17 +233,8 @@ curl -X POST http://localhost:3000/api/notification \ | API Server | [http://localhost:3000](http://localhost:3000) | Notification API | | Admin Dashboard | [http://localhost:3002](http://localhost:3002) | Monitoring & management | | Grafana | [http://localhost:3001](http://localhost:3001) | Log visualization | -| Kafka UI | [http://localhost:8080](http://localhost:8080) | Kafka topic monitoring | -### Add More Plugins -```bash -# Add Gmail email provider -npx @simplens/config-gen generate @simplens/nodemailer-gmail -c simplens.config.yaml - -# List available plugins -npx @simplens/config-gen list --official -``` > For production deployments, see the [Self-Hosting Guide](https://simplens.in/docs/core/self-hosting) which covers distributed deployments, cloud infrastructure integration, and security hardening. @@ -266,35 +244,6 @@ npx @simplens/config-gen list --official SimpleNS provides a comprehensive, authenticated REST API that powers both external clients and the Admin Dashboard. Because the core engine is decoupled from the UI, every action possible in the dashboard can be automated or integrated via REST endpoints. -| Category | Route | Method | Description | -|---|---|---|---| -| **System** | `/api` | `GET` | API Server status | -| | `/api/health` | `GET` | Health check probe (Docker / K8s) | -| **Notifications** | `/api/notification` | `POST` | Dispatch single notification | -| | `/api/notification/batch` | `POST` | Dispatch batch notification | -| | `/api/notification` | `GET` | Notification endpoint health | -| **Management** | `/api/notifications` | `GET` | Search & filter notification history | -| | `/api/notifications/recent` | `GET` | Activity feed of latest notifications | -| | `/api/notifications/:id` | `GET` | Fetch single notification details | -| | `/api/notifications/:id/retry` | `POST` | Reset failed notification for retry | -| | `/api/notifications/:id` | `DELETE` | Delete notification record | -| **Templates** | `/api/templates/create` | `POST` | Create reusable notification template | -| | `/api/templates` | `GET` | List all templates (or filter by package) | -| | `/api/templates/:template_id` | `GET` | Fetch template by ID | -| | `/api/templates/:template_id` | `PUT` | Update notification template | -| | `/api/templates/:template_id` | `DELETE` | Delete notification template | -| **Plugins** | `/api/plugins` | `GET` | Dynamic plugin metadata & schemas | -| **Alerts** | `/api/alerts` | `GET` | List system alerts | -| | `/api/alerts/:id/resolve` | `POST` | Resolve alert and retry notification | -| | `/api/alerts/bulk-resolve` | `POST` | Bulk resolve alerts and retry | -| | `/api/alerts/:id` | `DELETE` | Dismiss alert without retrying | -| **Dashboard** | `/api/dashboard/stats` | `GET` | Notification counts by status & channel | -| | `/api/dashboard/trends` | `GET` | Historical time-series trend data | -| **Admin Channels** | `/api/admin-channels/providers` | `GET` | Available alerting channel providers | -| | `/api/admin-channels/validate` | `POST` | Validate channel configuration schema | -| | `/api/admin-channels/test` | `POST` | Test live webhook or bot connection | -| | `/api/admin-channels` | `GET` / `POST` | List or create admin alert channels | -| | `/api/admin-channels/:id` | `GET` / `PATCH` / `DELETE` | Manage individual admin alert channels | > 📖 **Interactive API Collection**: A complete [Bruno](https://www.usebruno.com/) collection is available in [`/api-docs`](./api-docs) with preconfigured environments and sample requests for all endpoints. @@ -311,17 +260,17 @@ The Admin Dashboard is an independent Next.js application built strictly as a ** - 🏢 **Headless Ready**: Because the dashboard is fully decoupled, you can operate SimpleNS headless in backend environments, deploy custom frontends, or interact via CI/CD and AI tools without running the dashboard. **Key Dashboard Features:** -- 🏠 **Dashboard Home** — Status overview powered by `/api/dashboard/stats` -- 📡 **Channel Cards** — Visual status for each configured delivery channel -- 📋 **Events Explorer** — Paginated event table with live filtering via `/api/notifications` -- 🔴 **Failed Events** — Dedicated view with one-click retries via `/api/notifications/:id/retry` -- 🚨 **Alerts** — System health and orphaned recovery alerts via `/api/alerts` -- 📈 **Analytics** — 24h, 7d, and 30d time-series trends via `/api/dashboard/trends` -- 🔌 **Plugins** — Dynamic form generation based on runtime schemas from `/api/plugins` -- 🔧 **Payload Studio** — Interactive schema explorer for constructing valid payloads -- 🔔 **Admin Alert Channels** — Live test and configure Discord/Telegram alerts via `/api/admin-channels` -- 📝 **Notification Templates** — Full lifecycle management and live preview via `/api/templates` - +- **Dashboard Home** — Status overview powered by `/api/dashboard/stats` +- **Channel Cards** — Visual status for each configured delivery channel +- **Events Explorer** — Paginated event table with live filtering via `/api/notifications` +- **Failed Events** — Dedicated view with one-click retries via `/api/notifications/:id/retry` +- **Alerts** — System health and orphaned recovery alerts via `/api/alerts` +- **Analytics** — 24h, 7d, and 30d time-series trends via `/api/dashboard/trends` +- **Plugins** — Manage official, community and custom plugins, setup providers with credentials and setup channel routing with an array of fallback providers +- **Payload Studio** — Interactive schema explorer for constructing valid payloads +- **Admin Alert Channels** — Live test and configure Discord/Telegram alerts via `/api/admin-channels` +- **Notification Templates** — Full lifecycle management and live preview via `/api/templates` +- **API Key** - Create and manage multiple API keys, track their usage and so on --- ## License diff --git a/api-docs/.env.example b/api-docs/.env.example index 3fb5e47..12085d7 100644 --- a/api-docs/.env.example +++ b/api-docs/.env.example @@ -1,5 +1,6 @@ BASE_URL=http://localhost:3000 -NS_API_KEY= +API_KEY= +API_KEY_ID= TO_EMAIL= TO_EMAIL_2= CHANNEL=mock @@ -11,4 +12,10 @@ ALERT_ID= ADMIN_CHANNEL_ID= DISCORD_WEBHOOK_URL= TELEGRAM_BOT_TOKEN= -TELEGRAM_CHAT_ID= \ No newline at end of file +TELEGRAM_CHAT_ID= +ADMIN_USERNAME=admin +ADMIN_PASSWORD=SecureAdminPassword123! +PROVIDER_ID=mock-provider-1 +ROUTING_CHANNEL=mock +NPM_AUTH_TOKEN= +NPM_REGISTRY_URL=https://registry.npmjs.org/ \ No newline at end of file diff --git a/api-docs/README.md b/api-docs/README.md index e0798e6..4be9e96 100644 --- a/api-docs/README.md +++ b/api-docs/README.md @@ -24,12 +24,13 @@ Refer to the [Bruno Secrets Management & DotEnv Documentation](https://docs.useb | Variable | Description | Example / Default | | :--- | :--- | :--- | | `BASE_URL` | Base HTTP URL of the SimpleNS API server | `http://localhost:3000` | -| `NS_API_KEY` | SimpleNS API Bearer Key for authentication | `ns_...` | +| `API_KEY` | SimpleNS API Bearer Key for authentication | `sns_...` | +| `API_KEY_ID` | MongoDB ObjectId or key_id of an API key | `64f...` | | `TO_EMAIL` | Sample recipient email address | `user@example.com` | | `TO_EMAIL_2` | Secondary recipient email address for batch requests | `user2@example.com` | | `CHANNEL` | Notification channel identifier | `mock` | | `PROVIDER` | Channel provider identifier | `mock` | -| `PACKAGE_NAME` | Plugin package name for templates | `simplens-plugin-mock` | +| `PACKAGE_NAME` | Plugin package name for templates and plugins | `simplens-plugin-mock` | | `TEMPLATE_ID` | Identifier of a notification template | `mock-sample-1` | | `NOTIFICATION_ID` | MongoDB ObjectId of a notification for management endpoints | `64f...` | | `ALERT_ID` | MongoDB ObjectId of an alert for alert endpoints | `64f...` | @@ -37,6 +38,32 @@ Refer to the [Bruno Secrets Management & DotEnv Documentation](https://docs.useb | `DISCORD_WEBHOOK_URL` | Discord webhook URL for testing admin channel alerts | `https://discord.com/api/webhooks/...` | | `TELEGRAM_BOT_TOKEN` | Telegram Bot token for testing admin channel alerts | `123456789:ABCdefGhI...` | | `TELEGRAM_CHAT_ID` | Telegram chat ID for testing admin channel alerts | `-1001234567890` | +| `ADMIN_USERNAME` | Administrator username for initial setup & verification | `admin` | +| `ADMIN_PASSWORD` | Administrator password for initial setup & verification | `SecureAdminPassword123!` | +| `PROVIDER_ID` | Identifier for configured provider instance | `mock-provider-1` | +| `ROUTING_CHANNEL` | Notification channel identifier for routing config | `mock` | +| `NPM_AUTH_TOKEN` | Private npm registry authentication token | `npm_...` | +| `NPM_REGISTRY_URL` | Custom or default npm registry URL | `https://registry.npmjs.org/` | + +--- + +## Authentication & Authorization + +SimpleNS supports dual authentication across all protected API routes: + +1. **Admin Dashboard (Browser Cookie)**: + - Authenticated using the HTTP-only `simplens_session` cookie. + - Automatically set by the Core API when calling `POST /api/admin/auth/signup` or `POST /api/admin/auth/login`. + - Cleared on `POST /api/admin/auth/logout`. + - Browsers automatically attach this cookie on every request without requiring client-side token storage. + +2. **Programmatic API Clients & Services (API Key or Bearer Token)**: + - Authenticated using dynamic API keys (`sns_live_...` or `sns_test_...`) created via the Dashboard or API. + - Passed via either header: + - `Authorization: Bearer ` + - `x-api-key: ` + +All protected routes in SimpleNS accept either authentication method interchangeably. --- @@ -47,13 +74,32 @@ Public server and health information (no authentication required). - **API Root**: `GET /api` - Basic server info. - **Health Check**: `GET /api/health` - Health status and timestamp for Docker/Kubernetes probes. -### 2. `notifications` (Protected) +### 2. `admin_auth` (Public) +Administrator setup, authentication, and session management. +- **Check Admin Auth Status**: `GET /api/admin/auth/status` - Check whether an administrator account has already been configured. +- **Signup Admin Account**: `POST /api/admin/auth/signup` - Configure the primary administrator account (only allowed once during setup). Sets `simplens_session` cookie. + ```json + { + "username": "{{process.env.ADMIN_USERNAME}}", + "password": "{{process.env.ADMIN_PASSWORD}}" + } + ``` +- **Login Admin Account**: `POST /api/admin/auth/login` - Verify administrator credentials and set the `simplens_session` HTTP-only cookie. + ```json + { + "username": "{{process.env.ADMIN_USERNAME}}", + "password": "{{process.env.ADMIN_PASSWORD}}" + } + ``` +- **Logout Admin Account**: `POST /api/admin/auth/logout` - Clear and invalidate the `simplens_session` cookie. + +### 3. `notifications` (Protected) Notification dispatch and status. - **Single Request**: `POST /api/notification` - Enqueue a single notification across one or more channels. - **Batch Request**: `POST /api/notification/batch` - Enqueue batch notifications across recipients. - **Check Notification Endpoint**: `GET /api/notification` - Check notification service status. -### 3. `notifications_management` (Protected) +### 4. `notifications_management` (Protected) Query, inspect, retry, and delete notification records. - **List Notifications**: `GET /api/notifications` - Paginated, sorted, and filtered notification listing. - **Get Recent Notifications**: `GET /api/notifications/recent` - Live activity feed of latest notifications. @@ -61,7 +107,7 @@ Query, inspect, retry, and delete notification records. - **Retry Notification**: `POST /api/notifications/:id/retry` - Reset a failed notification to pending for re-execution. - **Delete Notification**: `DELETE /api/notifications/:id` - Delete a notification record. -### 4. `notification_templates` (Protected) +### 5. `notification_templates` (Protected) Manage reusable message templates for plugins. - **Create Template**: `POST /api/templates/create` - Register a new notification template. - **List All Templates**: `GET /api/templates` - Retrieve all templates. @@ -70,23 +116,98 @@ Manage reusable message templates for plugins. - **Update Template**: `PUT /api/templates/:template_id` - Update an existing template. - **Delete Template**: `DELETE /api/templates/:template_id` - Remove a template. -### 5. `plugins` (Protected) -Channel and provider discovery. +### 6. `plugins` (Protected) +Channel discovery, installed npm plugins, public plugin catalog, npm registry authentication, and plugin lifecycle. - **Get Plugin Metadata**: `GET /api/plugins` - List registered channels, active providers, and content/credential schemas. +- **List Installed Plugins**: `GET /api/plugins/installed` - List all installed npm plugins in MongoDB. +- **List Plugin Catalog**: `GET /api/plugins/catalog/:category` - Fetch official or community plugin catalog (`official` | `community`). +- **Get npm Auth Status**: `GET /api/plugins/npm-auth` - Retrieve npm authentication status with masked tokens. +- **Save npm Auth Config**: `POST /api/plugins/npm-auth` - Save custom npm token and registry configuration. + ```json + { + "token": "{{process.env.NPM_AUTH_TOKEN}}", + "registry_url": "{{process.env.NPM_REGISTRY_URL}}" + } + ``` +- **Delete npm Auth Config**: `DELETE /api/plugins/npm-auth` - Remove configured npm registry credentials. +- **Install Plugin**: `POST /api/plugins/install` - Download, verify, and register an npm plugin package. + ```json + { + "package": "{{process.env.PACKAGE_NAME}}", + "version": "latest", + "auth": { + "token": "{{process.env.NPM_AUTH_TOKEN}}", + "registry_url": "{{process.env.NPM_REGISTRY_URL}}", + "save_token": true + } + } + ``` +- **Change Plugin Version**: `PUT /api/plugins/version` - Upgrade or downgrade an installed plugin version. + ```json + { + "package": "{{process.env.PACKAGE_NAME}}", + "version": "1.0.1" + } + ``` +- **Uninstall Plugin**: `DELETE /api/plugins/:package` - Uninstall and clean up a plugin package. + +### 7. `providers` (Protected) +Manage provider instances, test connections, and inspect/reset token-bucket rate limits. +- **List Providers**: `GET /api/providers` - Retrieve all configured provider instances. +- **Create Provider**: `POST /api/providers` - Register a new provider with encrypted credentials and rate limiting options. + ```json + { + "id": "{{process.env.PROVIDER_ID}}", + "plugin_name": "{{process.env.PACKAGE_NAME}}", + "credentials": { + "apiKey": "sample-provider-api-key" + }, + "options": { + "priority": 1, + "rateLimit": { + "maxTokens": 100, + "refillRate": 10, + "refillInterval": "second" + } + }, + "enabled": true + } + ``` +- **Get Provider by Id**: `GET /api/providers/:id` - Fetch provider by ID (with optional `include_decrypted=true/false`). +- **Update Provider**: `PUT /api/providers/:id` - Update provider credentials, priority, or rate limits. +- **Delete Provider**: `DELETE /api/providers/:id` - Remove a provider instance. +- **Test Provider Connection**: `POST /api/providers/test` - Test provider connectivity without persisting credentials. +- **Get Providers Rate Limits**: `GET /api/providers/rate-limits` - Real-time rate limiter status across all providers. +- **Get Provider Rate Limit**: `GET /api/providers/:id/rate-limit` - Real-time rate limiter tokens and telemetry for a specific provider. +- **Reset Provider Rate Limit**: `POST /api/providers/:id/rate-limit/reset` - Manually refill tokens and clear rate limiter backoffs. -### 6. `alerts` (Protected) +### 8. `channel_routing` (Protected) +Configure default providers, fallback provider chains, and Kafka partition count per notification channel. +- **List Channel Routings**: `GET /api/channels/routing` - List all channel routing configurations. +- **Get Channel Routing**: `GET /api/channels/routing/:channel` - Retrieve routing for a specific channel (e.g. `email`, `mock`). +- **Set Channel Routing**: `PUT /api/channels/routing/:channel` - Configure default provider, cascading fallback array, and partitions. + ```json + { + "default_provider_id": "{{process.env.PROVIDER_ID}}", + "fallback_provider_ids": [], + "partitions": 6 + } + ``` +- **Delete Channel Routing**: `DELETE /api/channels/routing/:channel` - Remove routing configuration for a channel. + +### 9. `alerts` (Protected) Manage system alerts triggered by failures or stuck workers. - **List Alerts**: `GET /api/alerts` - List unresolved alerts with pagination and type summary counts. - **Resolve Alert**: `POST /api/alerts/:id/resolve` - Mark alert as resolved and retry the underlying notification. - **Bulk Resolve Alerts**: `POST /api/alerts/bulk-resolve` - Resolve multiple alerts and retry notifications in batch. - **Dismiss Alert**: `DELETE /api/alerts/:id` - Dismiss an alert without retrying. -### 7. `dashboard` (Protected) +### 10. `dashboard` (Protected) Analytics and trends for the admin dashboard. - **Get Dashboard Stats**: `GET /api/dashboard/stats` - Notification counts grouped by status and channel. - **Get Dashboard Trends**: `GET /api/dashboard/trends` - Hourly or daily time-series counts (`24h`, `7d`, `30d`). -### 8. `admin_channels` (Protected) +### 11. `admin_channels` (Protected) Manage administrator alert channels (Discord, Telegram, etc.). - **List Admin Channel Providers**: `GET /api/admin-channels/providers` - Available provider integrations and schemas. - **Validate Admin Channel Config**: `POST /api/admin-channels/validate` - Validate channel credentials against schema without saving. @@ -95,4 +216,63 @@ Manage administrator alert channels (Discord, Telegram, etc.). - **Create Admin Channel**: `POST /api/admin-channels` - Create a new encrypted admin alert channel. - **Get Admin Channel by Id**: `GET /api/admin-channels/:id` - Fetch single channel details. - **Update Admin Channel**: `PATCH /api/admin-channels/:id` - Update channel name, active status, filters, or config. -- **Delete Admin Channel**: `DELETE /api/admin-channels/:id` - Remove an alert channel. \ No newline at end of file +- **Delete Admin Channel**: `DELETE /api/admin-channels/:id` - Remove an alert channel. + +### 12. `settings` (Protected) +Read, modify, and reset live operational configuration across services via Redis sync. +- **Get Operational Settings**: `GET /api/settings` - Retrieve current active settings for API, workers, retry, delayed poller, recovery, and logging. +- **Update Operational Settings**: `PUT /api/settings` - Update one or more configuration fields. + ```json + { + "api": { + "max_batch_req_limit": 1000 + }, + "worker": { + "outbox_poll_interval_ms": 5000, + "outbox_cleanup_interval_ms": 60000, + "outbox_batch_size": 100, + "outbox_retention_ms": 300000, + "outbox_claim_timeout_ms": 30000 + }, + "retry": { + "max_retry_count": 5, + "idempotency_ttl_seconds": 86400, + "processing_ttl_seconds": 120, + "rate_limit_retry_delay_ms": 5000 + }, + "delayed": { + "delayed_poll_interval_ms": 1000, + "delayed_batch_size": 10, + "max_poller_retries": 3 + }, + "recovery": { + "recovery_poll_interval_ms": 60000, + "processing_stuck_threshold_ms": 300000, + "pending_stuck_threshold_ms": 300000, + "recovery_batch_size": 50, + "recovery_claim_timeout_ms": 60000, + "cleanup_resolved_alerts_retention_ms": 86400000, + "cleanup_processed_status_outbox_retention_ms": 86400000 + }, + "logging": { + "log_level": "info", + "log_to_file": true + } + } + ``` +- **Reset Operational Settings**: `POST /api/settings/reset` - Reset operational settings to system defaults and broadcast to all services. + +### 13. `api_keys` (Protected) +Create, manage, inspect, and revoke programmatic API keys. +- **List API Keys**: `GET /api/keys` - List all active and revoked API keys. +- **Create API Key**: `POST /api/keys` - Generate a new scoped API key (`sns_live_...` or `sns_test_...`). + ```json + { + "name": "Integration Service Key", + "type": "live", + "scopes": ["*"] + } + ``` +- **Get API Key Usage**: `GET /api/keys/:id` - Inspect key metadata, usage counters, and last used timestamp. +- **Revoke API Key**: `POST /api/keys/:id/revoke` - Immediately deactivate an active API key. +- **Delete API Key**: `DELETE /api/keys/:id` - Permanently remove an API key record. \ No newline at end of file diff --git a/api-docs/admin_auth/Admin Login.bru b/api-docs/admin_auth/Admin Login.bru new file mode 100644 index 0000000..a641099 --- /dev/null +++ b/api-docs/admin_auth/Admin Login.bru @@ -0,0 +1,23 @@ +meta { + name: Login Admin Account + type: http + seq: 3 +} + +post { + url: {{process.env.BASE_URL}}/api/admin/auth/login + body: json + auth: none +} + +body:json { + { + "username": "{{process.env.ADMIN_USERNAME}}", + "password": "{{process.env.ADMIN_PASSWORD}}" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/admin_auth/Check Admin Auth Status.bru b/api-docs/admin_auth/Check Admin Auth Status.bru new file mode 100644 index 0000000..66cd7ae --- /dev/null +++ b/api-docs/admin_auth/Check Admin Auth Status.bru @@ -0,0 +1,16 @@ +meta { + name: Check Admin Auth Status + type: http + seq: 1 +} + +get { + url: {{process.env.BASE_URL}}/api/admin/auth/status + body: none + auth: none +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/admin_auth/Logout Admin Account.bru b/api-docs/admin_auth/Logout Admin Account.bru new file mode 100644 index 0000000..b66b19a --- /dev/null +++ b/api-docs/admin_auth/Logout Admin Account.bru @@ -0,0 +1,16 @@ +meta { + name: Logout Admin Account + type: http + seq: 4 +} + +post { + url: {{process.env.BASE_URL}}/api/admin/auth/logout + body: none + auth: none +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/admin_auth/Setup Admin Account.bru b/api-docs/admin_auth/Setup Admin Account.bru new file mode 100644 index 0000000..eefc07f --- /dev/null +++ b/api-docs/admin_auth/Setup Admin Account.bru @@ -0,0 +1,23 @@ +meta { + name: Signup Admin Account + type: http + seq: 2 +} + +post { + url: {{process.env.BASE_URL}}/api/admin/auth/signup + body: json + auth: none +} + +body:json { + { + "username": "{{process.env.ADMIN_USERNAME}}", + "password": "{{process.env.ADMIN_PASSWORD}}" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/admin_auth/folder.bru b/api-docs/admin_auth/folder.bru new file mode 100644 index 0000000..eb85e74 --- /dev/null +++ b/api-docs/admin_auth/folder.bru @@ -0,0 +1,7 @@ +meta { + name: admin_auth +} + +auth { + mode: none +} diff --git a/api-docs/api_keys/Create API Key.bru b/api-docs/api_keys/Create API Key.bru new file mode 100644 index 0000000..9057919 --- /dev/null +++ b/api-docs/api_keys/Create API Key.bru @@ -0,0 +1,24 @@ +meta { + name: Create API Key + type: http + seq: 2 +} + +post { + url: {{process.env.BASE_URL}}/api/keys + body: json + auth: inherit +} + +body:json { + { + "name": "Integration Test Key", + "type": "live", + "scopes": ["*"] + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/api_keys/Delete API Key.bru b/api-docs/api_keys/Delete API Key.bru new file mode 100644 index 0000000..5d681ee --- /dev/null +++ b/api-docs/api_keys/Delete API Key.bru @@ -0,0 +1,16 @@ +meta { + name: Delete API Key + type: http + seq: 5 +} + +delete { + url: {{process.env.BASE_URL}}/api/keys/{{process.env.API_KEY_ID}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/api_keys/Get API Key Usage.bru b/api-docs/api_keys/Get API Key Usage.bru new file mode 100644 index 0000000..538928b --- /dev/null +++ b/api-docs/api_keys/Get API Key Usage.bru @@ -0,0 +1,16 @@ +meta { + name: Get API Key Usage + type: http + seq: 3 +} + +get { + url: {{process.env.BASE_URL}}/api/keys/{{process.env.API_KEY_ID}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/api_keys/List API Keys.bru b/api-docs/api_keys/List API Keys.bru new file mode 100644 index 0000000..6242315 --- /dev/null +++ b/api-docs/api_keys/List API Keys.bru @@ -0,0 +1,16 @@ +meta { + name: List API Keys + type: http + seq: 1 +} + +get { + url: {{process.env.BASE_URL}}/api/keys + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/api_keys/Revoke API Key.bru b/api-docs/api_keys/Revoke API Key.bru new file mode 100644 index 0000000..1b9885c --- /dev/null +++ b/api-docs/api_keys/Revoke API Key.bru @@ -0,0 +1,16 @@ +meta { + name: Revoke API Key + type: http + seq: 4 +} + +post { + url: {{process.env.BASE_URL}}/api/keys/{{process.env.API_KEY_ID}}/revoke + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/api_keys/folder.bru b/api-docs/api_keys/folder.bru new file mode 100644 index 0000000..d227949 --- /dev/null +++ b/api-docs/api_keys/folder.bru @@ -0,0 +1,7 @@ +meta { + name: api_keys +} + +auth { + mode: inherit +} diff --git a/api-docs/channel_routing/Delete Channel Routing.bru b/api-docs/channel_routing/Delete Channel Routing.bru new file mode 100644 index 0000000..2e906b1 --- /dev/null +++ b/api-docs/channel_routing/Delete Channel Routing.bru @@ -0,0 +1,16 @@ +meta { + name: Delete Channel Routing + type: http + seq: 4 +} + +delete { + url: {{process.env.BASE_URL}}/api/channels/routing/{{process.env.ROUTING_CHANNEL}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/channel_routing/Get Channel Routing.bru b/api-docs/channel_routing/Get Channel Routing.bru new file mode 100644 index 0000000..a63cb4a --- /dev/null +++ b/api-docs/channel_routing/Get Channel Routing.bru @@ -0,0 +1,16 @@ +meta { + name: Get Channel Routing + type: http + seq: 2 +} + +get { + url: {{process.env.BASE_URL}}/api/channels/routing/{{process.env.ROUTING_CHANNEL}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/channel_routing/List Channel Routings.bru b/api-docs/channel_routing/List Channel Routings.bru new file mode 100644 index 0000000..032d6d4 --- /dev/null +++ b/api-docs/channel_routing/List Channel Routings.bru @@ -0,0 +1,16 @@ +meta { + name: List Channel Routings + type: http + seq: 1 +} + +get { + url: {{process.env.BASE_URL}}/api/channels/routing + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/channel_routing/Set Channel Routing.bru b/api-docs/channel_routing/Set Channel Routing.bru new file mode 100644 index 0000000..98555a8 --- /dev/null +++ b/api-docs/channel_routing/Set Channel Routing.bru @@ -0,0 +1,24 @@ +meta { + name: Set Channel Routing + type: http + seq: 3 +} + +put { + url: {{process.env.BASE_URL}}/api/channels/routing/{{process.env.ROUTING_CHANNEL}} + body: json + auth: inherit +} + +body:json { + { + "default_provider_id": "{{process.env.PROVIDER_ID}}", + "fallback_provider_ids": [], + "partitions": 6 + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/channel_routing/folder.bru b/api-docs/channel_routing/folder.bru new file mode 100644 index 0000000..f6dd828 --- /dev/null +++ b/api-docs/channel_routing/folder.bru @@ -0,0 +1,7 @@ +meta { + name: channel_routing +} + +auth { + mode: inherit +} diff --git a/api-docs/collection.bru b/api-docs/collection.bru index 22ec557..776f23a 100644 --- a/api-docs/collection.bru +++ b/api-docs/collection.bru @@ -3,5 +3,5 @@ auth { } auth:bearer { - token: {{process.env.NS_API_KEY}} + token: {{process.env.API_KEY}} } diff --git a/api-docs/plugins/Change Plugin Version.bru b/api-docs/plugins/Change Plugin Version.bru new file mode 100644 index 0000000..a308a48 --- /dev/null +++ b/api-docs/plugins/Change Plugin Version.bru @@ -0,0 +1,23 @@ +meta { + name: Change Plugin Version + type: http + seq: 8 +} + +put { + url: {{process.env.BASE_URL}}/api/plugins/version + body: json + auth: inherit +} + +body:json { + { + "package": "{{process.env.PACKAGE_NAME}}", + "version": "1.0.1" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/Delete npm Auth Config.bru b/api-docs/plugins/Delete npm Auth Config.bru new file mode 100644 index 0000000..e0033cc --- /dev/null +++ b/api-docs/plugins/Delete npm Auth Config.bru @@ -0,0 +1,21 @@ +meta { + name: Delete npm Auth Config + type: http + seq: 6 +} + +delete { + url: {{process.env.BASE_URL}}/api/plugins/npm-auth?id= + body: none + auth: inherit +} + +params:query { + ~id: + ~scope: +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/Get npm Auth Status.bru b/api-docs/plugins/Get npm Auth Status.bru new file mode 100644 index 0000000..436ccc5 --- /dev/null +++ b/api-docs/plugins/Get npm Auth Status.bru @@ -0,0 +1,16 @@ +meta { + name: Get npm Auth Status + type: http + seq: 4 +} + +get { + url: {{process.env.BASE_URL}}/api/plugins/npm-auth + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/Install Plugin.bru b/api-docs/plugins/Install Plugin.bru new file mode 100644 index 0000000..06c396c --- /dev/null +++ b/api-docs/plugins/Install Plugin.bru @@ -0,0 +1,28 @@ +meta { + name: Install Plugin + type: http + seq: 7 +} + +post { + url: {{process.env.BASE_URL}}/api/plugins/install + body: json + auth: inherit +} + +body:json { + { + "package": "{{process.env.PACKAGE_NAME}}", + "version": "latest", + "auth": { + "token": "{{process.env.NPM_AUTH_TOKEN}}", + "registry_url": "{{process.env.NPM_REGISTRY_URL}}", + "save_token": true + } + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/List Installed Plugins.bru b/api-docs/plugins/List Installed Plugins.bru new file mode 100644 index 0000000..f606027 --- /dev/null +++ b/api-docs/plugins/List Installed Plugins.bru @@ -0,0 +1,16 @@ +meta { + name: List Installed Plugins + type: http + seq: 2 +} + +get { + url: {{process.env.BASE_URL}}/api/plugins/installed + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/List Plugin Catalog.bru b/api-docs/plugins/List Plugin Catalog.bru new file mode 100644 index 0000000..f29d90f --- /dev/null +++ b/api-docs/plugins/List Plugin Catalog.bru @@ -0,0 +1,16 @@ +meta { + name: List Plugin Catalog + type: http + seq: 3 +} + +get { + url: {{process.env.BASE_URL}}/api/plugins/catalog/official + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/Save npm Auth Config.bru b/api-docs/plugins/Save npm Auth Config.bru new file mode 100644 index 0000000..bb02139 --- /dev/null +++ b/api-docs/plugins/Save npm Auth Config.bru @@ -0,0 +1,23 @@ +meta { + name: Save npm Auth Config + type: http + seq: 5 +} + +post { + url: {{process.env.BASE_URL}}/api/plugins/npm-auth + body: json + auth: inherit +} + +body:json { + { + "token": "{{process.env.NPM_AUTH_TOKEN}}", + "registry_url": "{{process.env.NPM_REGISTRY_URL}}" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/plugins/Uninstall Plugin.bru b/api-docs/plugins/Uninstall Plugin.bru new file mode 100644 index 0000000..74498ea --- /dev/null +++ b/api-docs/plugins/Uninstall Plugin.bru @@ -0,0 +1,16 @@ +meta { + name: Uninstall Plugin + type: http + seq: 9 +} + +delete { + url: {{process.env.BASE_URL}}/api/plugins/{{process.env.PACKAGE_NAME}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Create Provider.bru b/api-docs/providers/Create Provider.bru new file mode 100644 index 0000000..6a51e66 --- /dev/null +++ b/api-docs/providers/Create Provider.bru @@ -0,0 +1,35 @@ +meta { + name: Create Provider + type: http + seq: 2 +} + +post { + url: {{process.env.BASE_URL}}/api/providers + body: json + auth: inherit +} + +body:json { + { + "id": "{{process.env.PROVIDER_ID}}", + "plugin_name": "{{process.env.PACKAGE_NAME}}", + "credentials": { + "apiKey": "sample-provider-api-key" + }, + "options": { + "priority": 1, + "rateLimit": { + "maxTokens": 100, + "refillRate": 10, + "refillInterval": "second" + } + }, + "enabled": true + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Delete Provider.bru b/api-docs/providers/Delete Provider.bru new file mode 100644 index 0000000..f894def --- /dev/null +++ b/api-docs/providers/Delete Provider.bru @@ -0,0 +1,16 @@ +meta { + name: Delete Provider + type: http + seq: 5 +} + +delete { + url: {{process.env.BASE_URL}}/api/providers/{{process.env.PROVIDER_ID}} + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Get Provider Rate Limit.bru b/api-docs/providers/Get Provider Rate Limit.bru new file mode 100644 index 0000000..2bc4f8c --- /dev/null +++ b/api-docs/providers/Get Provider Rate Limit.bru @@ -0,0 +1,16 @@ +meta { + name: Get Provider Rate Limit + type: http + seq: 8 +} + +get { + url: {{process.env.BASE_URL}}/api/providers/{{process.env.PROVIDER_ID}}/rate-limit + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Get Provider by Id.bru b/api-docs/providers/Get Provider by Id.bru new file mode 100644 index 0000000..66c1226 --- /dev/null +++ b/api-docs/providers/Get Provider by Id.bru @@ -0,0 +1,20 @@ +meta { + name: Get Provider by Id + type: http + seq: 3 +} + +get { + url: {{process.env.BASE_URL}}/api/providers/{{process.env.PROVIDER_ID}}?include_decrypted=false + body: none + auth: inherit +} + +params:query { + include_decrypted: false +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Get Providers Rate Limits.bru b/api-docs/providers/Get Providers Rate Limits.bru new file mode 100644 index 0000000..c83de8c --- /dev/null +++ b/api-docs/providers/Get Providers Rate Limits.bru @@ -0,0 +1,16 @@ +meta { + name: Get Providers Rate Limits + type: http + seq: 7 +} + +get { + url: {{process.env.BASE_URL}}/api/providers/rate-limits + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/List Providers.bru b/api-docs/providers/List Providers.bru new file mode 100644 index 0000000..1d3fcb9 --- /dev/null +++ b/api-docs/providers/List Providers.bru @@ -0,0 +1,16 @@ +meta { + name: List Providers + type: http + seq: 1 +} + +get { + url: {{process.env.BASE_URL}}/api/providers + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Reset Provider Rate Limit.bru b/api-docs/providers/Reset Provider Rate Limit.bru new file mode 100644 index 0000000..6df0533 --- /dev/null +++ b/api-docs/providers/Reset Provider Rate Limit.bru @@ -0,0 +1,16 @@ +meta { + name: Reset Provider Rate Limit + type: http + seq: 9 +} + +post { + url: {{process.env.BASE_URL}}/api/providers/{{process.env.PROVIDER_ID}}/rate-limit/reset + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Test Provider Connection.bru b/api-docs/providers/Test Provider Connection.bru new file mode 100644 index 0000000..6be0172 --- /dev/null +++ b/api-docs/providers/Test Provider Connection.bru @@ -0,0 +1,26 @@ +meta { + name: Test Provider Connection + type: http + seq: 6 +} + +post { + url: {{process.env.BASE_URL}}/api/providers/test + body: json + auth: inherit +} + +body:json { + { + "plugin_name": "{{process.env.PACKAGE_NAME}}", + "credentials": { + "apiKey": "sample-provider-api-key" + }, + "options": {} + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/Update Provider.bru b/api-docs/providers/Update Provider.bru new file mode 100644 index 0000000..a8e8065 --- /dev/null +++ b/api-docs/providers/Update Provider.bru @@ -0,0 +1,30 @@ +meta { + name: Update Provider + type: http + seq: 4 +} + +put { + url: {{process.env.BASE_URL}}/api/providers/{{process.env.PROVIDER_ID}} + body: json + auth: inherit +} + +body:json { + { + "enabled": true, + "options": { + "priority": 2, + "rateLimit": { + "maxTokens": 200, + "refillRate": 20, + "refillInterval": "second" + } + } + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/providers/folder.bru b/api-docs/providers/folder.bru new file mode 100644 index 0000000..10dac8b --- /dev/null +++ b/api-docs/providers/folder.bru @@ -0,0 +1,7 @@ +meta { + name: providers +} + +auth { + mode: inherit +} diff --git a/api-docs/settings/Get Operational Settings.bru b/api-docs/settings/Get Operational Settings.bru new file mode 100644 index 0000000..ebd5cd2 --- /dev/null +++ b/api-docs/settings/Get Operational Settings.bru @@ -0,0 +1,16 @@ +meta { + name: Get Operational Settings + type: http + seq: 1 +} + +get { + url: {{process.env.BASE_URL}}/api/settings + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/settings/Reset Operational Settings.bru b/api-docs/settings/Reset Operational Settings.bru new file mode 100644 index 0000000..7f64191 --- /dev/null +++ b/api-docs/settings/Reset Operational Settings.bru @@ -0,0 +1,16 @@ +meta { + name: Reset Operational Settings + type: http + seq: 3 +} + +post { + url: {{process.env.BASE_URL}}/api/settings/reset + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/settings/Update Operational Settings.bru b/api-docs/settings/Update Operational Settings.bru new file mode 100644 index 0000000..59491b0 --- /dev/null +++ b/api-docs/settings/Update Operational Settings.bru @@ -0,0 +1,55 @@ +meta { + name: Update Operational Settings + type: http + seq: 2 +} + +put { + url: {{process.env.BASE_URL}}/api/settings + body: json + auth: inherit +} + +body:json { + { + "api": { + "max_batch_req_limit": 1000 + }, + "worker": { + "outbox_poll_interval_ms": 5000, + "outbox_cleanup_interval_ms": 60000, + "outbox_batch_size": 100, + "outbox_retention_ms": 300000, + "outbox_claim_timeout_ms": 30000 + }, + "retry": { + "max_retry_count": 5, + "idempotency_ttl_seconds": 86400, + "processing_ttl_seconds": 120, + "rate_limit_retry_delay_ms": 5000 + }, + "delayed": { + "delayed_poll_interval_ms": 1000, + "delayed_batch_size": 10, + "max_poller_retries": 3 + }, + "recovery": { + "recovery_poll_interval_ms": 60000, + "processing_stuck_threshold_ms": 300000, + "pending_stuck_threshold_ms": 300000, + "recovery_batch_size": 50, + "recovery_claim_timeout_ms": 60000, + "cleanup_resolved_alerts_retention_ms": 86400000, + "cleanup_processed_status_outbox_retention_ms": 86400000 + }, + "logging": { + "log_level": "info", + "log_to_file": true + } + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/api-docs/settings/folder.bru b/api-docs/settings/folder.bru new file mode 100644 index 0000000..c24c666 --- /dev/null +++ b/api-docs/settings/folder.bru @@ -0,0 +1,7 @@ +meta { + name: settings +} + +auth { + mode: inherit +} diff --git a/dashboard/.env.example b/dashboard/.env.example index 591ccd6..9c73113 100644 --- a/dashboard/.env.example +++ b/dashboard/.env.example @@ -1,11 +1,4 @@ -MONGO_URI= # mongodb connection uri +NODE_ENV= # set as production, development, test etc.. used for setting the secure propery for cookie PORT=3002 # port number -ADMIN_USERNAME= # admin username -ADMIN_PASSWORD= # admin password -AUTH_SECRET= # authentication secret -NS_API_KEY= # notification service api key -API_BASE_URL= # notification service base url -WEBHOOK_HOST= # webhook host -WEBHOOK_PORT= # webhook port +API_BASE_URL= # notification service base url (e.g. http://localhost:3000) BASE_PATH= # Base path for Next.js app (e.g., /dashboard). Leave empty for root path. -HTTPS_COOKIE= # Controls the storage of session cookies in http and https mode. (If set false, then the cookies can be stored in http connection) diff --git a/dashboard/app/api/auth/login/route.ts b/dashboard/app/api/auth/login/route.ts deleted file mode 100644 index f6dd3a5..0000000 --- a/dashboard/app/api/auth/login/route.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { NextResponse } from "next/server"; -import { createSession, validateCredentials } from "@/lib/session"; -import { getBasePath } from "@/lib/utils"; - -export async function POST(request: Request) { - try { - const body = await request.json(); - const { username, password } = body; - - if (!username || !password) { - return NextResponse.json( - { error: "Username and password are required" }, - { status: 400 } - ); - } - - const result = validateCredentials(username, password); - - if (!result.isValid || !result.userId) { - return NextResponse.json( - { error: "Invalid username or password" }, - { status: 401 } - ); - } - - await createSession(result.userId, username); - - const basePath = getBasePath(); - return NextResponse.json({ - success: true, - redirectUrl: `${basePath}/dashboard`, - }); - } catch (error) { - console.error("Login error:", error); - return NextResponse.json( - { error: "An error occurred during login" }, - { status: 500 } - ); - } -} diff --git a/dashboard/app/api/auth/logout/route.ts b/dashboard/app/api/auth/logout/route.ts deleted file mode 100644 index b018613..0000000 --- a/dashboard/app/api/auth/logout/route.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { NextResponse } from "next/server"; -import { destroySession } from "@/lib/session"; - -export async function POST() { - try { - await destroySession(); - - return NextResponse.json({ success: true }); - } catch (error) { - console.error("Logout error:", error); - return NextResponse.json( - { error: "An error occurred during logout" }, - { status: 500 } - ); - } -} diff --git a/dashboard/app/api/auth/session/route.ts b/dashboard/app/api/auth/session/route.ts deleted file mode 100644 index 7f79a0a..0000000 --- a/dashboard/app/api/auth/session/route.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { NextResponse } from "next/server"; -import { validateSession } from "@/lib/session"; - -export async function GET() { - try { - const session = await validateSession(); - - if (!session.isValid) { - return NextResponse.json({ - authenticated: false, - }); - } - - return NextResponse.json({ - authenticated: true, - user: session.user, - }); - } catch (error) { - console.error("Session check error:", error); - return NextResponse.json({ - authenticated: false, - }); - } -} diff --git a/dashboard/app/api/runtime-config/route.ts b/dashboard/app/api/runtime-config/route.ts deleted file mode 100644 index e2872ec..0000000 --- a/dashboard/app/api/runtime-config/route.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { NextResponse } from "next/server"; - -/** - * Runtime configuration API endpoint - * Returns the base path configured at runtime via environment variable - * This serves as a fallback if /runtime-config.js is not loaded - */ -export async function GET() { - return NextResponse.json({ - basePath: process.env.BASE_PATH || "", - }); -} - -export const dynamic = 'force-dynamic'; diff --git a/dashboard/app/api/webhook/route.ts b/dashboard/app/api/webhook/route.ts deleted file mode 100644 index 06c29e7..0000000 --- a/dashboard/app/api/webhook/route.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { NextResponse } from "next/server"; - -/** - * Webhook endpoint to receive notification delivery status callbacks. - * The worker already updates status in MongoDB, so we just acknowledge receipt. - */ -export async function POST(request: Request) { - try { - const payload = await request.json(); - - // Log the webhook callback for debugging (optional) - console.log("[Webhook] Received callback:", { - request_id: payload.request_id, - notification_id: payload.notification_id, - status: payload.status, - channel: payload.channel, - occurred_at: payload.occurred_at, - }); - - // Return 200 OK to acknowledge receipt - return NextResponse.json({ received: true }); - } catch (error) { - console.error("[Webhook] Error processing callback:", error); - return NextResponse.json({ received: false, error: "Invalid payload" }, { status: 400 }); - } -} diff --git a/dashboard/app/events/[id]/page.tsx b/dashboard/app/events/[id]/page.tsx index df5f451..6c39029 100644 --- a/dashboard/app/events/[id]/page.tsx +++ b/dashboard/app/events/[id]/page.tsx @@ -382,6 +382,88 @@ export default function EventDetailPage({ params }: PageProps) { + {/* Provider Execution History */} + + +
+
+ Provider Execution History + + Timeline of provider dispatch attempts, rate limit cascades, and failure reasons + +
+ {notification.provider_history && notification.provider_history.length > 0 && ( + + {notification.provider_history.length} {notification.provider_history.length === 1 ? 'attempt' : 'attempts'} + + )} +
+
+ + {(!notification.provider_history || notification.provider_history.length === 0) ? ( +
+ No intermediate provider cascade attempts recorded. +
+ ) : ( +
+ {notification.provider_history.map((attempt, index) => { + const isSuccess = attempt.status === 'delivered'; + const isRateLimit = attempt.status === 'rate_limited'; + + return ( +
+
+
+ {index + 1}. {attempt.provider} + + {isSuccess ? 'Delivered' : isRateLimit ? 'Rate Limited' : 'Failed'} + + {attempt.error_code && ( + + {attempt.error_code} + + )} + {attempt.duration_ms !== undefined && ( + + ({attempt.duration_ms}ms) + + )} +
+ {attempt.error_message && ( +

+ {attempt.error_message} +

+ )} +
+
+ + {format(new Date(attempt.attempted_at), "PPpp")} +
+
+ ); + })} +
+ )} +
+
+ {/* Error Message (if failed) */} {isFailed && notification.error_message && ( diff --git a/dashboard/app/keys/page.tsx b/dashboard/app/keys/page.tsx new file mode 100644 index 0000000..5d0ea54 --- /dev/null +++ b/dashboard/app/keys/page.tsx @@ -0,0 +1,598 @@ +"use client"; + +import { useState } from "react"; +import useSWR from "swr"; +import { DashboardLayout } from "@/components/layout/dashboard-layout"; +import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from "@/components/ui/dialog"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { + Key, + Plus, + Copy, + Check, + Trash2, + Ban, + BarChart3, + MoreVertical, + Activity, + CheckCircle2, +} from "lucide-react"; +import { toast } from "sonner"; +import { apiClient, apiKeyService } from "@/lib/api-client"; +import type { ApiKey, ApiKeyUsageDetailResponse, ApiKeyType } from "@/lib/types"; + +const fetcher = (url: string): Promise => apiClient.get(url) as unknown as Promise; + +export default function ApiKeysPage() { + const { data, isLoading, mutate } = useSWR<{ keys: ApiKey[] }>("/api/keys", fetcher); + const keys = data?.keys || []; + + // Create Modal state + const [createOpen, setCreateOpen] = useState(false); + const [newKeyName, setNewKeyName] = useState(""); + const [newKeyType, setNewKeyType] = useState("notification"); + const [isCreating, setIsCreating] = useState(false); + + // Reveal Key Modal state + const [revealedKey, setRevealedKey] = useState<{ name: string; raw_key: string } | null>(null); + const [copied, setCopied] = useState(false); + + // Action dialogs + const [revokeKeyId, setRevokeKeyId] = useState(null); + const [deleteKeyId, setDeleteKeyId] = useState(null); + + // Usage Details Sheet/Modal state + const [selectedUsage, setSelectedUsage] = useState(null); + + const handleCreateKey = async (e: React.FormEvent) => { + e.preventDefault(); + if (!newKeyName.trim()) { + toast.error("Please enter a key name"); + return; + } + + setIsCreating(true); + try { + const res = await apiKeyService.create({ name: newKeyName.trim(), type: newKeyType }); + toast.success("API key created successfully"); + setCreateOpen(false); + setNewKeyName(""); + setNewKeyType("notification"); + setRevealedKey({ name: res.key.name, raw_key: res.raw_key }); + mutate(); + } catch (err: unknown) { + console.error(err); + toast.error("Failed to create API key"); + } finally { + setIsCreating(false); + } + }; + + const handleCopyKey = () => { + if (!revealedKey) return; + navigator.clipboard.writeText(revealedKey.raw_key); + setCopied(true); + toast.success("API Key copied to clipboard"); + setTimeout(() => setCopied(false), 2000); + }; + + const handleRevoke = async () => { + if (!revokeKeyId) return; + try { + await apiKeyService.revoke(revokeKeyId); + toast.success("API key has been revoked"); + mutate(); + } catch (err) { + console.error(err); + toast.error("Failed to revoke API key"); + } finally { + setRevokeKeyId(null); + } + }; + + const handleDelete = async () => { + if (!deleteKeyId) return; + try { + await apiKeyService.delete(deleteKeyId); + toast.success("API key deleted permanently"); + mutate(); + } catch (err) { + console.error(err); + toast.error("Failed to delete API key"); + } finally { + setDeleteKeyId(null); + } + }; + + const handleViewUsage = async (keyId: string) => { + try { + const detail = await apiKeyService.getUsage(keyId); + setSelectedUsage(detail); + } catch (err) { + console.error(err); + toast.error("Failed to load key usage statistics"); + } + }; + + // Calculate metrics + const totalKeys = keys.length; + const activeKeys = keys.filter((k) => k.status === "active").length; + const totalNotifications = keys.reduce( + (sum, k) => sum + (k.usage?.total_notifications || 0), + 0 + ); + + return ( + +
+ {/* Actions Toolbar */} +
+ +
+ + {/* Metrics Cards */} +
+ + + Total API Keys + + + +
{totalKeys}
+

+ {activeKeys} active, {totalKeys - activeKeys} revoked +

+
+
+ + + + Total Ingested Notifications + + + +
{totalNotifications.toLocaleString()}
+

+ Processed across all generated API keys +

+
+
+
+ + {/* API Keys Table */} + + + Active Credentials + + API keys allow external systems and automation workers to interact with SimpleNS. + + + + {isLoading ? ( +
+ + + +
+ ) : keys.length === 0 ? ( +
+ +

No API keys found

+

Create an API key to allow external services to send notifications.

+
+ ) : ( +
+ + + + Name + Key Prefix + Type + Status + Usage (Notifications) + Channel Breakdown + Last Used + Actions + + + + {keys.map((key) => { + const channels = key.usage?.by_channel || {}; + const channelEntries = Object.entries(channels); + + return ( + + + {key.name} + + + + {key.key_prefix} + + + + {key.type === "admin" ? ( + + Admin (Full Access) + + ) : ( + + Notification + + )} + + + {key.status === "active" ? ( + + Active + + ) : ( + + Revoked + + )} + + + + {(key.usage?.total_notifications || 0).toLocaleString()} + + + ({key.usage?.total_requests || 0} reqs) + + + +
+ {channelEntries.length > 0 ? ( + channelEntries.map(([ch, count]) => ( + + {ch}: {count} + + )) + ) : ( + None + )} +
+
+ + {key.usage?.last_used_at + ? new Date(key.usage.last_used_at).toLocaleString() + : "Never"} + + + + + + + + handleViewUsage(key.key_id)}> + + View Details & Stats + + {key.status === "active" && ( + setRevokeKeyId(key.key_id)} + > + + Revoke Key + + )} + setDeleteKeyId(key.key_id)} + > + + Delete Key + + + + +
+ ); + })} +
+
+
+ )} +
+
+ + {/* Create Key Modal */} + + +
+ + Create External API Key + + Enter a descriptive name for this API key. The key secret will be displayed once after creation. + + +
+
+ + setNewKeyName(e.target.value)} + required + autoFocus + /> +
+ +
+ +
+
setNewKeyType("notification")} + className={`cursor-pointer rounded-lg border p-3 transition-colors ${ + newKeyType === "notification" + ? "border-primary bg-primary/5 text-primary" + : "border-border hover:bg-muted/50" + }`} + > +
+ Notification + {newKeyType === "notification" && } +
+

+ Can send notifications and check delivery status. +

+
+
setNewKeyType("admin")} + className={`cursor-pointer rounded-lg border p-3 transition-colors ${ + newKeyType === "admin" + ? "border-primary bg-primary/5 text-primary" + : "border-border hover:bg-muted/50" + }`} + > +
+ Admin (Full Access) + {newKeyType === "admin" && } +
+

+ Full access to management endpoints and notifications. +

+
+
+
+
+ + + + +
+
+
+ + {/* Key Reveal Dialog */} + { + if (!open) setRevealedKey(null); + }} + > + + + + + API Key Created: {revealedKey?.name} + + + Please copy your API key secret immediately. For security reasons, you will not be able to view it again. + + + +
+
+ + +
+ +
+ Keep this key confidential. External applications should pass this key in the Authorization: Bearer <key> header. +
+
+ + + + +
+
+ + {/* Usage Detail Dialog */} + !open && setSelectedUsage(null)}> + + + + + API Key Usage: {selectedUsage?.key.name} + + + Real-time delivery outcomes and volume tracking for this key. + + + + {selectedUsage && ( +
+
+
+
Total Ingested
+
+ {selectedUsage.key.usage?.total_notifications.toLocaleString()} +
+
+
+
Total API Requests
+
+ {selectedUsage.key.usage?.total_requests.toLocaleString()} +
+
+
+ +
+

+ Delivery Status Breakdown +

+
+
+
+ {selectedUsage.status_breakdown.delivered} +
+
Delivered
+
+
+
+ {selectedUsage.status_breakdown.failed} +
+
Failed
+
+
+
+ {selectedUsage.status_breakdown.processing} +
+
Processing
+
+
+
+ {selectedUsage.status_breakdown.pending} +
+
Pending
+
+
+
+ +
+

+ Notifications by Channel +

+
+ {Object.entries(selectedUsage.key.usage?.by_channel || {}).length > 0 ? ( + Object.entries(selectedUsage.key.usage.by_channel).map(([channel, count]) => ( +
+ {channel} + {count.toLocaleString()} +
+ )) + ) : ( +

No channel activity recorded yet.

+ )} +
+
+
+ )} + + + + +
+
+ + {/* Revoke Confirmation Dialog */} + !open && setRevokeKeyId(null)}> + + + Revoke API Key? + + Once revoked, any external service or tool attempting to authenticate with this API key will be immediately blocked with 401 Unauthorized. + + + + Cancel + + Revoke Key + + + + + + {/* Delete Confirmation Dialog */} + !open && setDeleteKeyId(null)}> + + + Delete API Key Permanently? + + This will permanently remove the API key record. This action cannot be undone. + + + + Cancel + + Delete Permanently + + + + +
+
+ ); +} diff --git a/dashboard/app/layout.tsx b/dashboard/app/layout.tsx index b53af14..5bffc45 100644 --- a/dashboard/app/layout.tsx +++ b/dashboard/app/layout.tsx @@ -3,7 +3,6 @@ import { Geist, Geist_Mono } from "next/font/google"; import Script from "next/script"; import { Toaster } from "@/components/ui/sonner"; import { ThemeProvider } from "@/components/theme-provider"; -import { AuthProvider } from "@/components/auth-provider"; import { GlassmorphismProvider } from "@/components/glassmorphism-provider"; import { SidebarPinProvider } from "@/components/sidebar-pin-provider"; import { TourProvider } from "@/components/tour/tour-provider"; @@ -38,23 +37,21 @@ export default function RootLayout({ - - - - - - {children} - - - - - - + + + + + {children} + + + + + ); diff --git a/dashboard/app/login/page.tsx b/dashboard/app/login/page.tsx index e640518..a831855 100644 --- a/dashboard/app/login/page.tsx +++ b/dashboard/app/login/page.tsx @@ -11,7 +11,7 @@ import { AlertTriangle, Eye, EyeOff, Loader2, Lock, User } from "lucide-react"; import { ElegantShape } from "@/components/elegant-shape"; import { withBasePath } from "@/lib/utils"; import { authService } from "@/lib/api-client"; -import { useState } from "react"; +import { useEffect, useState } from "react"; export default function LoginPage() { const router = useRouter(); @@ -21,6 +21,21 @@ export default function LoginPage() { const [isLoading, setIsLoading] = useState(false); const [showPassword, setShowPassword] = useState(false); + useEffect(() => { + const checkStatus = async () => { + try { + const status = await authService.getAuthStatus(); + if (!status.isConfigured) { + router.replace(withBasePath("/setup")); + } + } catch (err) { + console.warn("Failed to check auth configuration status on login page", err); + } + }; + + checkStatus(); + }, [router]); + const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); setError(""); @@ -30,8 +45,7 @@ export default function LoginPage() { const data = await authService.login({ username, password }); setIsLoading(false); // Redirect to dashboard - router.push(data.redirectUrl || withBasePath(`/dashboard`)); - router.refresh(); + window.location.href = data.redirectUrl || withBasePath(`/dashboard`); return; } catch (err) { const error = err as Error; diff --git a/dashboard/app/plugins/page.tsx b/dashboard/app/plugins/page.tsx index decf9d4..575e6a5 100644 --- a/dashboard/app/plugins/page.tsx +++ b/dashboard/app/plugins/page.tsx @@ -1,315 +1,52 @@ "use client"; +import { useState } from "react"; import { DashboardLayout } from "@/components/layout/dashboard-layout"; -import { PageToolbar, PageToolbarSection, PageToolbarSpacer } from "@/components/ui/page-toolbar"; -import { Card, CardContent } from "@/components/ui/card"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { Skeleton } from "@/components/ui/skeleton"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import useSWR from "swr"; -import { Puzzle, RefreshCw, Mail, MessageCircle, TestTube2, Zap, User, FileText } from "lucide-react"; -import { useMemo, useState } from "react"; -import { apiClient } from "@/lib/api-client"; -import { ProviderMetadata, PluginMetadata as PluginsResponse } from "@/lib/types"; - -const fetcher = (url: string): Promise => apiClient.get(url) as unknown as Promise; - -// Channel-specific colors and icons -const channelConfig: Record; color: string }> = { - email: { - icon: Mail, - color: "text-blue-500", - }, - whatsapp: { - icon: MessageCircle, - color: "text-green-500", - }, - mock: { - icon: TestTube2, - color: "text-purple-500", - }, - sms: { - icon: MessageCircle, - color: "text-orange-500", - }, -}; - -const getChannelConfig = (channel: string) => { - return channelConfig[channel] || { - icon: Zap, - color: "text-green-500", - }; -}; - -interface PluginCardProps { - provider: ProviderMetadata; - channel: string; - isDefault: boolean; -} - -function PluginCard({ provider, channel, isDefault }: PluginCardProps) { - const config = getChannelConfig(channel); - const ChannelIcon = config.icon; - - return ( - - {/* Default Badge */} - {isDefault && ( -
- - Default - -
- )} - -
- {/* Header */} -
-
- -
-
-

- {provider.displayName || provider.name} -

-

- ID: {provider.id} -

-
-
- - {/* Channel & Priority */} -
-
- - Channel: - {channel} -
-
- Priority: - {provider.priority} -
-
- - {/* Recipient Fields */} - {provider.recipientFields?.length > 0 && ( -
-
- - Recipient Fields: -
-
- {provider.recipientFields.map((field) => ( - - {field.name}{field.required && "*"} - - ))} -
-
- )} - - {/* Content Fields */} - {provider.contentFields?.length > 0 && ( -
-
- - Content Fields: -
-
- {provider.contentFields.map((field) => ( - - {field.name}{field.required && "*"} - - ))} -
-
- )} - - {/* Description */} - {provider.description && ( -

- {provider.description} -

- )} -
-
- ); -} - -function PluginCardSkeleton() { - return ( - -
-
- -
- - -
- -
-
-
- ); -} +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Server, Package, GitBranch } from "lucide-react"; +import { ProvidersTab } from "@/components/plugins/providers-tab"; +import { InstalledPluginsTab } from "@/components/plugins/installed-plugins-tab"; +import { ChannelRoutingTab } from "@/components/plugins/channel-routing-tab"; export default function PluginsPage() { - const [channelFilter, setChannelFilter] = useState("all"); - - const { data, isLoading, error, mutate } = useSWR( - "/api/plugins", - fetcher, - { refreshInterval: 30000 } - ); - - // Flatten channels into a list of providers with channel info - const allProviders = useMemo(() => { - if (!data?.channels) return []; - - const providers: Array<{ provider: ProviderMetadata; channel: string; isDefault: boolean }> = []; - - for (const [channel, channelData] of Object.entries(data.channels)) { - for (const provider of channelData.providers) { - providers.push({ - provider, - channel, - isDefault: channelData.default === provider.id - }); - } - } - - return providers; - }, [data]); - - // Get available channels for filter - const availableChannels = useMemo(() => { - if (!data?.channels) return []; - return Object.keys(data.channels); - }, [data]); - - // Filter providers by selected channel - const filteredProviders = useMemo(() => { - if (channelFilter === "all") return allProviders; - return allProviders.filter(p => p.channel === channelFilter); - }, [allProviders, channelFilter]); - - const channelCount = data?.channels ? Object.keys(data.channels).length : 0; - - return ( - -
- {/* Toolbar */} - - -
- - - {isLoading ? ( - - ) : ( - `${allProviders.length} plugin(s) installed • ${channelCount} channel(s) available` - )} - -
-
- - - - - - - -
- - {/* Loading State */} - {isLoading && ( -
- {[...Array(3)].map((_, i) => ( - - ))} -
- )} - - {/* Error State */} - {error && ( - - -

Failed to load plugins. Make sure the API is running.

-
-
- )} - - {/* Empty State */} - {data && allProviders.length === 0 && ( - - -
- -
-

No Plugins Installed

-

- Get started by installing your first notification plugin. -

- -
-
- )} - - {/* Plugin Cards */} - {data && filteredProviders.length > 0 && ( -
- {filteredProviders.map(({ provider, channel, isDefault }) => ( - - ))} -
- )} -
-
- ); + const [activeTab, setActiveTab] = useState("plugins"); + + return ( + +
+ {/* 3-Tab Interface */} + + + + + Installed Plugins + + + + Providers + + + + Channel Routing + + + + + + + + + + + + + + + +
+
+ ); } diff --git a/dashboard/app/settings/page.tsx b/dashboard/app/settings/page.tsx index 293d94f..df4d40f 100644 --- a/dashboard/app/settings/page.tsx +++ b/dashboard/app/settings/page.tsx @@ -1,30 +1,143 @@ "use client"; +import { useEffect, useState, useMemo } from "react"; import { DashboardLayout } from "@/components/layout/dashboard-layout"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Separator } from "@/components/ui/separator"; import { Badge } from "@/components/ui/badge"; import { Switch } from "@/components/ui/switch"; import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { useGlassmorphism } from "@/components/glassmorphism-provider"; import { useTour } from "@/components/tour/tour-provider"; -import { RotateCcw } from "lucide-react"; +import { + ChevronDown, + Clock, + Cpu, + Database, + Layers, + Loader2, + RefreshCw, + RotateCcw, + Save, + ShieldAlert, + Sliders, + Sparkles, +} from "lucide-react"; +import { settingsService } from "@/lib/api-client"; +import type { OperationalSettings } from "@/lib/types"; +import { toast } from "sonner"; import packageJson from "../../package.json"; export default function SettingsPage() { const { enabled, setEnabled } = useGlassmorphism(); const { restartTour } = useTour(); + const [settings, setSettings] = useState(null); + const [initialSettings, setInitialSettings] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [isSaving, setIsSaving] = useState(false); + const [isResetting, setIsResetting] = useState(false); + const [showAdvanced, setShowAdvanced] = useState(false); + + // Compute whether operational settings have been modified from original loaded state + const hasChanges = useMemo(() => { + if (!settings || !initialSettings) return false; + return JSON.stringify(settings) !== JSON.stringify(initialSettings); + }, [settings, initialSettings]); + + // Fetch operational settings on mount + useEffect(() => { + const fetchSettings = async () => { + try { + const res = await settingsService.get(); + if (res.success && res.settings) { + setSettings(res.settings); + setInitialSettings(res.settings); + } + } catch (err) { + console.error("Failed to load settings", err); + toast.error("Failed to load system settings from server"); + } finally { + setIsLoading(false); + } + }; + + fetchSettings(); + }, []); + + const handleSave = async () => { + if (!settings) return; + setIsSaving(true); + try { + const res = await settingsService.update(settings); + if (res.success && res.settings) { + setSettings(res.settings); + setInitialSettings(res.settings); + toast.success("Settings updated and synchronized across all instances!"); + } + } catch (err: unknown) { + const errorObj = err as Error; + toast.error(errorObj.message || "Failed to update settings"); + } finally { + setIsSaving(false); + } + }; + + const handleReset = async () => { + if (!confirm("Are you sure you want to reset all operational settings to system defaults?")) { + return; + } + + setIsResetting(true); + try { + const res = await settingsService.reset(); + if (res.success && res.settings) { + setSettings(res.settings); + setInitialSettings(res.settings); + toast.success("Operational settings reset to system defaults!"); + } + } catch (err: unknown) { + const errorObj = err as Error; + toast.error(errorObj.message || "Failed to reset settings"); + } finally { + setIsResetting(false); + } + }; + + const updateGroupField = ( + group: keyof OperationalSettings, + field: string, + value: unknown + ) => { + if (!settings) return; + setSettings({ + ...settings, + [group]: { + ...settings[group], + [field]: value, + }, + }); + }; + return ( -
- - - Appearance - Visual effects and UI rendering +
+ {/* 1. Appearance & Tour */} + + + + + Appearance & Tour + + + Visual styling layers and interactive guided walkthroughs +
@@ -54,7 +167,7 @@ export default function SettingsPage() { variant="outline" size="sm" onClick={restartTour} - className="gap-2" + className="gap-2 text-xs" > Restart Tour @@ -63,48 +176,661 @@ export default function SettingsPage() { - - {/* System Info */} - - - System Information - Current system configuration + {/* 2. System Info */} + + + + + System Information + + Current host deployment details
- Dashboard Version + Dashboard Version {packageJson.version}
- Environment + Runtime Mode {process.env.NODE_ENV === "production" ? "Production" : "Development"}
- Database - MongoDB + Configuration Store + + MongoDB (system_configs) & Redis Pub/Sub +
- {/* About */} - - - About SimpleNS - Notification service information - - -

- SimpleNS is an open-source, self-hosted notification orchestration engine that manages - delivery workflows—retries, scheduling, crash recovery, and scaling—while delegating - the actual sending to plugins. Build your own providers or use community plugins to - support any channel: Email, SMS, WhatsApp, Push, and beyond. -

-
-
+ {/* 3. Advanced Settings (Operational Engine Configurations) */} +
+ + + {showAdvanced && ( +
+ {/* Action Bar inside Advanced Settings */} +
+
+
+

Live Engine Tuning

+ {hasChanges && ( + + Unsaved Changes + + )} +
+

+ Changes synchronize immediately across all running containers without service restarts. +

+
+
+ + +
+
+ + {isLoading ? ( +
+ +

Loading system settings...

+
+ ) : !settings ? ( +
+

Failed to load configuration.

+ +
+ ) : ( +
+ {/* 1. Outbox Background Worker */} + + + + + Outbox Worker + + + Configures the background outbox polling, batch processing, and event cleanup. + + + +
+
+ + + updateGroupField( + "worker", + "outbox_poll_interval_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Safe: 500 - 60,000 +
+ +
+ + + updateGroupField( + "worker", + "outbox_batch_size", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Safe: 1 - 2,000 +
+
+ +
+
+ + + updateGroupField( + "worker", + "outbox_cleanup_interval_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Safe: 5k - 600k +
+ +
+ + + updateGroupField( + "worker", + "outbox_retention_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Published retention +
+
+ +
+ + + updateGroupField( + "worker", + "outbox_claim_timeout_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Threshold before an abandoned lock is reclaimed by peers + +
+
+
+ + {/* 2. Retry & Idempotency */} + + + + + Retry & Idempotency + + + Failure retries, deduplication caching, and execution locks. + + + +
+ + + updateGroupField( + "retry", + "max_retry_count", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Max attempts before permanently marking as failed + +
+ +
+ + + updateGroupField( + "retry", + "idempotency_ttl_seconds", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + How long deduplication keys are held in Redis (86400s = 24h) + +
+ +
+ + + updateGroupField( + "retry", + "processing_ttl_seconds", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Active processing mutex duration (default: 120s) + +
+ +
+ + + updateGroupField( + "retry", + "rate_limit_retry_delay_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Delay when all cascade providers are rate-limited (Safe: 100 - 60,000 ms) + +
+
+
+ + {/* 3. Delayed Processor */} + + + + + Delayed Processor + + + Parameters for scheduled and deferred notification poller. + + + +
+
+ + + updateGroupField( + "delayed", + "delayed_poll_interval_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Safe: 200 - 30,000 +
+ +
+ + + updateGroupField( + "delayed", + "delayed_batch_size", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Safe: 1 - 500 +
+
+ +
+ + + updateGroupField( + "delayed", + "max_poller_retries", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Attempts before moving an unroutable scheduled event to DLQ + +
+
+
+ + {/* 4. Recovery & Health Service */} + + + + + Recovery & Audits + + + Detects orphaned notifications, resolves stuck states, and manages cleanups. + + + +
+
+ + + updateGroupField( + "recovery", + "recovery_poll_interval_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 60,000 +
+ +
+ + + updateGroupField( + "recovery", + "recovery_batch_size", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 50 +
+
+ +
+
+ + + updateGroupField( + "recovery", + "processing_stuck_threshold_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 300,000 +
+ +
+ + + updateGroupField( + "recovery", + "pending_stuck_threshold_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 300,000 +
+
+ +
+ + + updateGroupField( + "recovery", + "recovery_claim_timeout_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 60,000 (instance claim lock) +
+ +
+
+ + + updateGroupField( + "recovery", + "cleanup_resolved_alerts_retention_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 86,400,000 (24h) +
+ +
+ + + updateGroupField( + "recovery", + "cleanup_processed_status_outbox_retention_ms", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + Default: 86,400,000 (24h) +
+
+
+
+ + {/* 5. System, API & Logging */} + + + + + API & Observability + + + API request boundaries and Winston logger dynamics across all containers. + + + +
+
+ + + updateGroupField( + "api", + "max_batch_req_limit", + parseInt(e.target.value) || 0 + ) + } + className="h-9 font-mono text-xs" + /> + + Max items per batch notification API call + +
+ +
+ + + + Immediately switches log level with zero restarts + +
+ +
+
+ +

+ Store rotating JSON log files in /logs directory +

+
+
+ + updateGroupField("logging", "log_to_file", val) + } + /> +
+
+
+
+
+
+ )} +
+ )} +
); diff --git a/dashboard/app/setup/page.tsx b/dashboard/app/setup/page.tsx new file mode 100644 index 0000000..ae22d60 --- /dev/null +++ b/dashboard/app/setup/page.tsx @@ -0,0 +1,1222 @@ +"use client"; + +import { useRouter } from "next/navigation"; +import { motion, AnimatePresence } from "motion/react"; +import Image from "next/image"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { + AlertTriangle, + ArrowLeft, + ArrowRight, + Bot, + Check, + CheckCircle2, + Clock, + Download, + Eye, + EyeOff, + HardDrive, + KeyRound, + Layers, + Loader2, + Lock, + Package, + Puzzle, + RefreshCw, + Server, + ShieldCheck, + SkipForward, + Sparkles, + User, + Zap, +} from "lucide-react"; +import { ElegantShape } from "@/components/elegant-shape"; +import { withBasePath } from "@/lib/utils"; +import { authService, pluginService, providerService } from "@/lib/api-client"; +import type { InstalledPlugin, PluginCatalogItem } from "@/lib/types"; +import { useEffect, useState } from "react"; +import { toast } from "sonner"; + +type SetupStep = 1 | 2 | 3 | 4; +type PluginPhase = "install" | "configure" | "complete"; + +// Fallback official plugins if public catalog CDN is temporarily unreachable +const FALLBACK_CATALOG: PluginCatalogItem[] = [ + { + name: "Nodemailer Gmail", + package: "@simplens/nodemailer-gmail", + description: "Send email notifications via Gmail SMTP using Nodemailer.", + }, + { + name: "Resend", + package: "@simplens/resend", + description: "Modern email delivery API for developers using Resend.", + }, + { + name: "Mock Provider", + package: "@simplens/mock", + description: "A mock notification provider for testing, verification, and local development.", + }, +]; + +export default function SetupPage() { + const router = useRouter(); + const [currentStep, setCurrentStep] = useState(1); + + // Form states + const [username, setUsername] = useState("admin"); + const [password, setPassword] = useState(""); + const [confirmPassword, setConfirmPassword] = useState(""); + const [error, setError] = useState(""); + const [isLoading, setIsLoading] = useState(false); + const [isCheckingStatus, setIsCheckingStatus] = useState(true); + const [showPassword, setShowPassword] = useState(false); + const [showConfirmPassword, setShowConfirmPassword] = useState(false); + + // Step 4 Plugin & Provider states + const [pluginPhase, setPluginPhase] = useState("install"); + const [catalog, setCatalog] = useState([]); + const [isCatalogLoading, setIsCatalogLoading] = useState(false); + const [installedPackages, setInstalledPackages] = useState>(new Set()); + const [newlyInstalledPlugins, setNewlyInstalledPlugins] = useState([]); + const [installingPkgs, setInstallingPkgs] = useState>(new Set()); + + // Helper to sanitize provider IDs without slashes ('/') + const sanitizeProviderId = (rawName: string): string => { + return rawName + .replace(/^@/, "") + .replace(/[/\\_ ]/g, "-") + .replace(/^plugin-/, "") + .replace(/-+/g, "-") + .toLowerCase() + + "-provider"; + }; + + // Provider configuration state + const [configureIndex, setConfigureIndex] = useState(0); + const [providerId, setProviderId] = useState(""); + const [credentials, setCredentials] = useState>({}); + const [isSavingProvider, setIsSavingProvider] = useState(false); + const [providerError, setProviderError] = useState(""); + + // On mount, check if admin credentials have already been configured + useEffect(() => { + const checkStatus = async () => { + try { + const status = await authService.getAuthStatus(); + if (status.isConfigured) { + router.replace(withBasePath("/login")); + return; + } + } catch (err) { + console.warn("Failed to check auth configuration status", err); + } finally { + setIsCheckingStatus(false); + } + }; + + checkStatus(); + }, [router]); + + // Password strength calculation + const hasMinLength = password.length >= 8; + const hasMaxLength = password.length <= 64; + const hasUpper = /[A-Z]/.test(password); + const hasLower = /[a-z]/.test(password); + const hasNumber = /[0-9]/.test(password); + const hasSpecial = /[^A-Za-z0-9]/.test(password); + + const score = [ + hasMinLength && hasMaxLength, + hasUpper && hasLower, + hasNumber, + hasSpecial, + ].filter(Boolean).length; + + const strengthLabel = + password.length === 0 ? "" : + score <= 1 ? "Weak" : + score === 2 ? "Fair" : + score === 3 ? "Good" : "Strong"; + + const strengthBarColor = + score <= 1 ? "bg-red-500" : + score === 2 ? "bg-amber-500" : + score === 3 ? "bg-blue-500" : "bg-emerald-500"; + + const strengthTextColor = + score <= 1 ? "text-red-500 dark:text-red-400" : + score === 2 ? "text-amber-500 dark:text-amber-400" : + score === 3 ? "text-blue-500 dark:text-blue-400" : "text-emerald-500 dark:text-emerald-400"; + + const passwordsMatch = password.length > 0 && password === confirmPassword; + + // Load plugin catalog when entering step 4 + const loadCatalog = async () => { + setIsCatalogLoading(true); + try { + const data = await pluginService.listCatalog("official"); + if (Array.isArray(data) && data.length > 0) { + setCatalog(data); + } else { + setCatalog(FALLBACK_CATALOG); + } + } catch { + setCatalog(FALLBACK_CATALOG); + } finally { + setIsCatalogLoading(false); + } + }; + + const handleSkipToDashboard = () => { + window.location.href = withBasePath("/dashboard"); + }; + + const handleAdminSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + setError(""); + + const trimmedUser = username.trim(); + if (trimmedUser.length < 3 || trimmedUser.length > 50) { + setError("Username must be between 3 and 50 characters."); + return; + } + + if (password.length < 8) { + setError("Password must be at least 8 characters long."); + return; + } + + if (password.length > 64) { + setError("Password cannot exceed 64 characters."); + return; + } + + if (password !== confirmPassword) { + setError("Passwords do not match."); + return; + } + + setIsLoading(true); + + try { + await authService.signup({ username: trimmedUser, password }); + + // Check if any plugins are already installed + try { + const installedRes = await pluginService.listInstalled(); + if (installedRes.plugins && installedRes.plugins.length > 0) { + window.location.href = withBasePath("/dashboard"); + return; + } + } catch { + // If listing plugins fails, proceed to step 4 as fallback + } + + // No plugins installed: advance to Step 4 (Discover & Install Plugins) + setIsLoading(false); + setCurrentStep(4); + setPluginPhase("install"); + void loadCatalog(); + } catch (err) { + const errorObj = err as Error; + setError(errorObj.message || "An error occurred during setup. Please try again."); + setIsLoading(false); + } + }; + + const handleInstallPlugin = async (item: PluginCatalogItem) => { + setInstallingPkgs((prev) => new Set(prev).add(item.package)); + try { + const res = await pluginService.install(item.package); + setInstalledPackages((prev) => new Set(prev).add(item.package)); + if (res.plugin) { + setNewlyInstalledPlugins((prev) => { + if (prev.some((p) => p.name === res.plugin.name)) return prev; + return [...prev, res.plugin]; + }); + } + toast.success(`Installed ${item.name} successfully!`); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : "Failed to install plugin"; + toast.error(msg); + } finally { + setInstallingPkgs((prev) => { + const next = new Set(prev); + next.delete(item.package); + return next; + }); + } + }; + + const handleStartConfigure = () => { + if (newlyInstalledPlugins.length === 0) { + handleSkipToDashboard(); + return; + } + setConfigureIndex(0); + const first = newlyInstalledPlugins[0]; + setProviderId(sanitizeProviderId(first.name)); + setCredentials({}); + setProviderError(""); + setPluginPhase("configure"); + }; + + const handleSaveProvider = async (e: React.FormEvent) => { + e.preventDefault(); + setProviderError(""); + + const currentPlugin = newlyInstalledPlugins[configureIndex]; + if (!currentPlugin) { + handleSkipToDashboard(); + return; + } + + const trimmedId = providerId.trim().replace(/[/\\ ]/g, "-"); + if (!trimmedId) { + setProviderError("Provider identifier is required."); + return; + } + if (trimmedId.includes("/")) { + setProviderError("Provider identifier cannot contain slashes ('/')."); + return; + } + + setIsSavingProvider(true); + + try { + await providerService.create({ + id: trimmedId, + plugin_name: currentPlugin.name, + credentials, + enabled: true, + }); + + toast.success(`Configured provider for ${currentPlugin.manifest?.displayName || currentPlugin.name}!`); + + // Move to next plugin or finish + if (configureIndex + 1 < newlyInstalledPlugins.length) { + const nextIndex = configureIndex + 1; + const nextPlugin = newlyInstalledPlugins[nextIndex]; + setConfigureIndex(nextIndex); + setProviderId(sanitizeProviderId(nextPlugin.name)); + setCredentials({}); + setProviderError(""); + } else { + setPluginPhase("complete"); + } + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : "Failed to configure provider"; + setProviderError(msg); + } finally { + setIsSavingProvider(false); + } + }; + + const handleSkipCurrentProvider = () => { + if (configureIndex + 1 < newlyInstalledPlugins.length) { + const nextIndex = configureIndex + 1; + const nextPlugin = newlyInstalledPlugins[nextIndex]; + setConfigureIndex(nextIndex); + setProviderId(sanitizeProviderId(nextPlugin.name)); + setCredentials({}); + setProviderError(""); + } else { + setPluginPhase("complete"); + } + }; + + if (isCheckingStatus) { + return ( +
+ +
+ ); + } + + const steps = [ + { number: 1, title: "Introduction", desc: "What is SimpleNS?" }, + { number: 2, title: "Capabilities", desc: "Features & Architecture" }, + { number: 3, title: "Administrator", desc: "Create Master Account" }, + { number: 4, title: "Plugins", desc: "Optional Provider Setup" }, + ]; + + const currentConfiguringPlugin = newlyInstalledPlugins[configureIndex]; + + return ( +
+ {/* Left Column: Brand & Interactive Progression Showcase */} +
+ {/* Background Ambient Shapes */} +
+ + +
+ + {/* Top Logo & Brand Tag */} +
+
+ SimpleNS Logo +
+
+ + First-Time Workspace Onboarding +
+
+ + {/* Step Timeline Indicator */} +
+
+ {steps.map((step) => { + const isCompleted = currentStep > step.number; + const isActive = currentStep === step.number; + + return ( + + ); + })} +
+ + {/* Step Highlight Preview Box */} +
+
+ {currentStep === 1 && } + {currentStep === 2 && } + {currentStep === 3 && } + {currentStep === 4 && } + + {currentStep === 1 && "Orchestration, Not Delivery"} + {currentStep === 2 && "Modular, Scalable & Extensible"} + {currentStep === 3 && "Secure Master Account"} + {currentStep === 4 && "Optional Provider Connectors"} + +
+

+ {currentStep === 1 && + "SimpleNS manages the complex notification lifecycle (retries, rate limiting, and recovery) while delegating actual sending to lightweight plugins."} + {currentStep === 2 && + "Enjoy complete data sovereignty, AI integration via Model Context Protocol (MCP), independent channel scaling, and zero per-notification vendor fees."} + {currentStep === 3 && + "Master administrator credentials are cryptographically protected and stored securely in MongoDB for complete operational authority."} + {currentStep === 4 && + "Install delivery provider plugins now or explore the full catalog at any time inside the dashboard."} +

+
+
+ + {/* Bottom Footer Note */} +
+ SimpleNS • Fast, Resilient, Open Architecture +
+
+ + {/* Right Column: Step Content Area */} +
+ {/* Mobile Top Header */} +
+ SimpleNS Logo +
+ Step {currentStep} of 4 +
+
+ +
+ + {/* ================= STEP 1: WHAT IS SIMPLENS ================= */} + {currentStep === 1 && ( + +
+
+ + Step 1 of 4 • Introduction +
+

+ What is SimpleNS? +

+

+ SimpleNS (Simple Notification System) is a self-hosted notification orchestration engine that solves your notification infrastructure once and for all. +

+
+ + {/* Core Concept Callout */} +
+
+ + Core Philosophy: Orchestration, Not Delivery +
+

+ SimpleNS handles the complex, heavy-lifting infrastructure logic—exponential backoff retries, rate limiting, delayed queues, idempotency deduplication, and crash recovery—while delegating actual delivery to lightweight, swappable plugins. +

+
+ + {/* 3 Essential Pillars */} +
+
+
+ +
+

+ 100% Self-Hosted +

+

+ Runs on your infrastructure. Maintain total data sovereignty and customer privacy with zero per-notification vendor fees. +

+
+ +
+
+ +
+

+ Plugin-Based +

+

+ Swap delivery providers (Email, SMS, Push, Chat) at any time without modifying a single line of application code. +

+
+ +
+
+ +
+

+ Zero Lock-In +

+

+ Standard REST APIs, webhook status callbacks, and open-source architecture that puts you in full control. +

+
+
+ + {/* Step 1 Navigation */} +
+ +
+
+ )} + + {/* ================= STEP 2: WHAT IT CAN DO ================= */} + {currentStep === 2 && ( + +
+
+ + Step 2 of 4 • Core Features +
+

+ What Can SimpleNS Do? +

+

+ A powerful, developer-friendly toolkit engineered for reliable notifications, effortless scaling, and modern AI automation. +

+
+ + {/* 6 High-Level Features Grid */} +
+ {/* Feature 1: Extensibility */} +
+
+ +
+
+

Modular Extensibility

+

+ Install official or community plugins for Email (SMTP/SES/Resend), SMS (Twilio/SNS), Push, Discord, Telegram, and Webhooks. +

+
+
+ + {/* Feature 2: MCP Server & AI */} +
+
+ +
+
+
+

AI & MCP Server Support

+ Native +
+

+ Built-in Model Context Protocol (MCP) server allows AI agents (Claude, Gemini, Cursor) to send notifications and resolve alerts autonomously. +

+
+
+ + {/* Feature 3: Scalability */} +
+
+ +
+
+

Horizontal Scalability

+

+ Scale notification workers independently per channel. Effortlessly absorb millions of deliveries with zero bottlenecks. +

+
+
+ + {/* Feature 4: Self-Hosting */} +
+
+ +
+
+

Self-Hosted & Private

+

+ Deploy effortlessly with Docker Compose or Kubernetes. All sensitive customer credentials and logs stay in your private network. +

+
+
+ + {/* Feature 5: Scheduled & Delayed Queueing */} +
+
+ +
+
+

Scheduled & Delayed Queue

+

+ Queue notifications for future campaigns, set reminders, or delay dispatch with millisecond precision and automatic retry fallbacks. +

+
+
+ + {/* Feature 6: Built-in Resilience */} +
+
+ +
+
+

Resilience & Recovery

+

+ Automated exponential backoff retries, provider rate limiting, ghost detection, and 1-click admin retry resolution. +

+
+
+
+ + {/* Step 2 Navigation */} +
+ + +
+
+ )} + + {/* ================= STEP 3: ADMIN SETUP ================= */} + {currentStep === 3 && ( + +
+
+ + Step 3 of 4 • Master Account +
+

+ Create Administrator +

+

+ Configure your master administrator credentials to secure and manage this SimpleNS instance. +

+
+ + {/* Setup Form */} +
+ {error && ( +
+ + {error} +
+ )} + + {/* Username */} +
+ + setUsername(e.target.value)} + required + minLength={3} + maxLength={50} + disabled={isLoading} + className="h-11 bg-background text-sm font-medium" + /> +

+ Between 3 and 50 characters. +

+
+ + {/* Password */} +
+ +
+ setPassword(e.target.value)} + required + minLength={8} + maxLength={64} + disabled={isLoading} + className="h-11 bg-background text-sm font-medium pr-10" + /> + +
+ + {/* Minimalist Password Strength Bar */} + {password.length > 0 && ( +
+
+ Password strength + + {strengthLabel} + +
+
+
+
+
+ )} + +

+ 8 to 64 characters with letters, numbers, and symbols. +

+
+ + {/* Confirm Password */} +
+ +
+ setConfirmPassword(e.target.value)} + required + minLength={8} + maxLength={64} + disabled={isLoading} + className="h-11 bg-background text-sm font-medium pr-10" + /> + +
+ {confirmPassword.length > 0 && ( +

+ {passwordsMatch ? ( + <> + Passwords match + + ) : ( + "Passwords do not match" + )} +

+ )} +
+ + {/* Step 3 Navigation Buttons */} +
+ + + +
+ + + )} + + {/* ================= STEP 4: OPTIONAL PLUGINS & PROVIDERS ================= */} + {currentStep === 4 && ( + + {/* Top Banner with Skip button */} +
+
+ + Admin Created • Optional Setup +
+ +
+ + {/* ================= PHASE 4A: DISCOVER & INSTALL ================= */} + {pluginPhase === "install" && ( +
+
+

+ Install Notification Plugins +

+

+ SimpleNS uses plugins to deliver messages to real-world services. Install one or more official plugins to get started, or skip and install them later in the dashboard. +

+
+ + {isCatalogLoading ? ( +
+ + Loading plugin catalog... +
+ ) : ( +
+ {catalog.map((item) => { + const isInstalled = installedPackages.has(item.package); + const isInstalling = installingPkgs.has(item.package); + + return ( +
+
+
+ + + {item.name} + + + {item.package} + +
+

+ {item.description} +

+
+ + +
+ ); + })} +
+ )} + + {/* Bottom Action Bar */} +
+ + + +
+
+ )} + + {/* ================= PHASE 4B: CONFIGURE PROVIDER ================= */} + {pluginPhase === "configure" && currentConfiguringPlugin && ( +
+
+
+ + Provider {configureIndex + 1} of {newlyInstalledPlugins.length} + + + {currentConfiguringPlugin.manifest?.displayName || currentConfiguringPlugin.name} + +
+

+ Configure Provider Credentials +

+

+ Set up connection credentials for this provider, or skip to configure it later in the dashboard. +

+
+ +
+ {providerError && ( +
+ + {providerError} +
+ )} + + {/* Provider ID */} +
+ + setProviderId(e.target.value.replace(/[/\\ ]/g, "-"))} + placeholder="my-provider-id" + required + disabled={isSavingProvider} + className="h-10 text-sm font-mono" + /> +

+ Unique identifier without slashes (e.g. gmail-provider). +

+
+ + {/* Required Credentials from Manifest */} + {currentConfiguringPlugin.manifest?.requiredCredentials && + currentConfiguringPlugin.manifest.requiredCredentials.length > 0 ? ( +
+ + Required Credentials + + {currentConfiguringPlugin.manifest.requiredCredentials.map((credKey) => ( +
+ + + setCredentials((prev) => ({ + ...prev, + [credKey]: e.target.value, + })) + } + disabled={isSavingProvider} + className="h-10 text-sm" + /> +
+ ))} +
+ ) : ( + /* Default generic API Key / Secret for mock or simple providers */ +
+ + + setCredentials((prev) => ({ + ...prev, + apiKey: e.target.value, + })) + } + disabled={isSavingProvider} + className="h-10 text-sm" + /> +
+ )} + + {/* Provider Buttons */} +
+ + + +
+
+
+ )} + + {/* ================= PHASE 4C: COMPLETE ================= */} + {pluginPhase === "complete" && ( +
+
+ +
+ +
+

+ You're All Set! +

+

+ Your master account and initial providers are configured. You are ready to start sending notifications with SimpleNS. +

+
+ +
+ +
+
+ )} +
+ )} + +
+ + {/* Bottom Step Dots on Mobile */} +
+ {steps.map((s) => ( +
s.number + ? "w-2 bg-primary/50" + : "w-2 bg-muted" + }`} + /> + ))} +
+
+
+ ); +} diff --git a/dashboard/components/auth-provider.tsx b/dashboard/components/auth-provider.tsx deleted file mode 100644 index f612a4a..0000000 --- a/dashboard/components/auth-provider.tsx +++ /dev/null @@ -1,82 +0,0 @@ -"use client"; - -import { createContext, useContext, useEffect, useState, useCallback } from "react"; -import { useRouter } from "next/navigation"; -import { withBasePath } from "@/lib/utils"; -import { authService } from "@/lib/api-client"; - -interface User { - id: string; - username: string; -} - -interface AuthContextType { - user: User | null; - isAuthenticated: boolean; - isLoading: boolean; - logout: () => Promise; - refreshSession: () => Promise; -} - -const AuthContext = createContext(undefined); - -export function AuthProvider({ children }: { children: React.ReactNode }) { - const router = useRouter(); - const [user, setUser] = useState(null); - const [isLoading, setIsLoading] = useState(true); - - const refreshSession = useCallback(async () => { - try { - const data = await authService.getSession(); - - if (data.authenticated && data.user) { - setUser(data.user); - } else { - setUser(null); - } - } catch (error) { - console.error("Failed to fetch session:", error); - setUser(null); - } finally { - setIsLoading(false); - } - }, []); - - const logout = useCallback(async () => { - try { - await authService.logout(); - setUser(null); - router.push(withBasePath(`/login`)); - router.refresh(); - } catch (error) { - console.error("Logout failed:", error); - window.location.href = withBasePath(`/login`); - } - }, [router]); - - useEffect(() => { - refreshSession(); - }, [refreshSession]); - - return ( - - {children} - - ); -} - -export function useAuth() { - const context = useContext(AuthContext); - if (context === undefined) { - throw new Error("useAuth must be used within an AuthProvider"); - } - return context; -} diff --git a/dashboard/components/dashboard/stats-cards.tsx b/dashboard/components/dashboard/stats-cards.tsx index e4e34fd..559d984 100644 --- a/dashboard/components/dashboard/stats-cards.tsx +++ b/dashboard/components/dashboard/stats-cards.tsx @@ -337,7 +337,7 @@ export function StatsGrid() {

No channels active yet.

-

Install plugins to see channel metrics.

+

Send channel-specific notifications to see channel metrics.

)} diff --git a/dashboard/components/layout/dashboard-layout.tsx b/dashboard/components/layout/dashboard-layout.tsx index ece91f5..5f29cce 100644 --- a/dashboard/components/layout/dashboard-layout.tsx +++ b/dashboard/components/layout/dashboard-layout.tsx @@ -17,6 +17,7 @@ import { FileText, Code, BellRing, + Key, } from "lucide-react"; import { Sidebar, @@ -81,6 +82,7 @@ const navCategories: NavCategory[] = [ { label: "System", items: [ + { title: "API Keys", href: "/keys", icon: Key }, { title: "Admin Alerts", href: "/admin-alerts", icon: BellRing }, ], }, diff --git a/dashboard/components/plugins/channel-routing-tab.tsx b/dashboard/components/plugins/channel-routing-tab.tsx new file mode 100644 index 0000000..3c67156 --- /dev/null +++ b/dashboard/components/plugins/channel-routing-tab.tsx @@ -0,0 +1,416 @@ +"use client"; + +import { useState } from "react"; +import useSWR from "swr"; +import { channelRoutingService, providerService } from "@/lib/api-client"; +import type { ChannelRoutingDto, ProviderDto } from "@/lib/types"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + GitBranch, + RefreshCw, + Edit2, + Plus, + ArrowRight, + Cpu, + Loader2, + AlertCircle, + X, + Trash2, +} from "lucide-react"; +import { toast } from "sonner"; + +export function ChannelRoutingTab() { + const { + data: routingData, + isLoading: routingLoading, + mutate: mutateRouting, + } = useSWR<{ routings: ChannelRoutingDto[] }>("/api/channels/routing", () => + channelRoutingService.list() + ); + + const { data: providersData } = useSWR<{ providers: ProviderDto[] }>("/api/providers", () => + providerService.list() + ); + + const [modalOpen, setModalOpen] = useState(false); + const [editingRouting, setEditingRouting] = useState(null); + + // Form State + const [channel, setChannel] = useState(""); + const [defaultProviderId, setDefaultProviderId] = useState(""); + const [fallbackIds, setFallbackIds] = useState([]); + const [partitions, setPartitions] = useState(6); + const [saving, setSaving] = useState(false); + const [deletingChannel, setDeletingChannel] = useState(null); + + const allProviders = providersData?.providers || []; + const routings = routingData?.routings || []; + + // Filter providers by selected channel + const channelProviders = allProviders.filter((p) => !channel || p.channel === channel); + + const handleOpenEdit = (route: ChannelRoutingDto) => { + setEditingRouting(route); + setChannel(route.channel); + setDefaultProviderId(route.default_provider_id); + setFallbackIds(route.fallback_provider_ids || []); + setPartitions(route.partitions || 6); + setModalOpen(true); + }; + + const handleOpenCreate = () => { + setEditingRouting(null); + setChannel(""); + setDefaultProviderId(""); + setFallbackIds([]); + setPartitions(6); + setModalOpen(true); + }; + + const handleAddFallback = (provId: string) => { + if (!provId || provId === defaultProviderId || fallbackIds.includes(provId)) return; + setFallbackIds([...fallbackIds, provId]); + }; + + const handleRemoveFallback = (index: number) => { + setFallbackIds(fallbackIds.filter((_, i) => i !== index)); + }; + + const handleSave = async (e: React.FormEvent) => { + e.preventDefault(); + if (!channel.trim()) { + toast.error("Channel name is required"); + return; + } + if (!defaultProviderId.trim()) { + toast.error("Default primary provider is required"); + return; + } + + setSaving(true); + try { + await channelRoutingService.set(channel.trim(), { + default_provider_id: defaultProviderId.trim(), + fallback_provider_ids: fallbackIds, + partitions, + }); + + toast.success(`Routing for channel '${channel}' updated`); + setModalOpen(false); + mutateRouting(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to update channel routing"; + toast.error(message); + } finally { + setSaving(false); + } + }; + + const handleDelete = async (route: ChannelRoutingDto) => { + if (!window.confirm(`Delete the routing for channel '${route.channel}'?`)) return; + + setDeletingChannel(route.channel); + try { + await channelRoutingService.delete(route.channel); + toast.success(`Routing for channel '${route.channel}' deleted`); + mutateRouting(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to delete channel routing"; + toast.error(message); + } finally { + setDeletingChannel(null); + } + }; + + return ( +
+ {/* Header */} +
+
+

Channel Routing & Kafka Scaling

+

+ Configure primary and cascading fallback providers per channel, and scale topic partitions. +

+
+
+ + +
+
+ + {/* Routing Cards Grid */} + {routingLoading ? ( +
+ {[1, 2].map((i) => ( + + + + + + + + ))} +
+ ) : routings.length === 0 ? ( + + + +
+

No channel routings configured

+

+ Map channels to primary and cascading fallback providers. +

+
+ +
+
+ ) : ( +
+ {routings.map((route) => ( + + +
+
+ + {route.channel} + +
+
+ + {route.partitions || 6} Partitions +
+
+ + {/* Routing Cascade Chain */} +
+
+ Cascading Failover Route +
+
+ + Primary: {route.default_provider_id} + + + {route.fallback_provider_ids && route.fallback_provider_ids.length > 0 ? ( + route.fallback_provider_ids.map((fId, idx) => ( +
+ + + Fallback {idx + 1}: {fId} + +
+ )) + ) : ( + + (No fallbacks configured) + + )} +
+
+ +
+ + +
+
+
+ ))} +
+ )} + + {/* Edit/Configure Routing Dialog */} + + +
+ + {editingRouting ? "Edit Channel Routing" : "Configure Channel"} + + Configure primary provider, ordered fallback failover cascade, and Kafka partition target. + + + +
+
+ + setChannel(e.target.value.toLowerCase())} + disabled={!!editingRouting} + required + /> +
+ +
+ + +
+ + {/* Cascading Fallback Array */} +
+ +

+ If the primary provider fails with a non-retryable error, SimpleNS tries each fallback in order. +

+ + {fallbackIds.length > 0 && ( +
+ {fallbackIds.map((fId, idx) => ( +
+ + {idx + 1}. {fId} + + +
+ ))} +
+ )} + +
+ +
+
+ + {/* Kafka Partitions Scaling */} +
+
+ + + Min: {editingRouting?.partitions || 1} + +
+ setPartitions(parseInt(e.target.value, 10) || 1)} + required + /> +

+ + Partitions can only be scaled up, never decreased. +

+
+
+ + + + + +
+
+
+
+ ); +} diff --git a/dashboard/components/plugins/installed-plugins-tab.tsx b/dashboard/components/plugins/installed-plugins-tab.tsx new file mode 100644 index 0000000..13bc004 --- /dev/null +++ b/dashboard/components/plugins/installed-plugins-tab.tsx @@ -0,0 +1,930 @@ +"use client"; + +import { useMemo, useState } from "react"; +import useSWR from "swr"; +import { pluginService } from "@/lib/api-client"; +import type { InstalledPlugin, PluginCatalogItem, InstallPluginPayload, NpmAuthStatus } from "@/lib/types"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Download, Package, RefreshCw, Trash2, ArrowUpDown, Loader2, ExternalLink, Info, Search, Lock, ShieldCheck, Eye, EyeOff, Check } from "lucide-react"; +import { toast } from "sonner"; + +export function InstalledPluginsTab() { + const { data, error, isLoading, mutate } = useSWR<{ plugins: InstalledPlugin[] }>( + "/api/plugins/installed", + () => pluginService.listInstalled() + ); + + const [installModalOpen, setInstallModalOpen] = useState(false); + const [catalogTab, setCatalogTab] = useState<"official" | "community" | "custom">("official"); + const [catalogSearch, setCatalogSearch] = useState(""); + const [selectedVersions, setSelectedVersions] = useState>({}); + const [installingPkgs, setInstallingPkgs] = useState>(new Set()); + + // Custom Package Form State + const [customPkg, setCustomPkg] = useState(""); + const [customVersion, setCustomVersion] = useState(""); + const [isPrivate, setIsPrivate] = useState(false); + const [npmToken, setNpmToken] = useState(""); + const [showToken, setShowToken] = useState(false); + const [enterpriseRegistryUrl, setEnterpriseRegistryUrl] = useState(""); + const [saveToken, setSaveToken] = useState(true); + const [overrideToken, setOverrideToken] = useState(false); + const [isInstallingCustom, setIsInstallingCustom] = useState(false); + + const { data: npmAuth, mutate: mutateNpmAuth } = useSWR( + installModalOpen ? "/api/plugins/npm-auth" : null, + () => pluginService.getNpmAuth() + ); + + const detectedScope = useMemo(() => { + const match = customPkg.trim().match(/^(@[a-zA-Z0-9~_.-]+)\//); + return match ? match[1].toLowerCase() : undefined; + }, [customPkg]); + + const matchedAuth = useMemo(() => { + if (!npmAuth?.is_configured) return null; + const regList = npmAuth.registries || []; + + // 1. Check scope match + if (detectedScope) { + const scopeMatch = regList.find( + (r) => r.scope?.toLowerCase() === detectedScope + ); + if (scopeMatch) return scopeMatch; + } + + // 2. Check registry URL match if entered + const regUrl = enterpriseRegistryUrl.trim(); + if (regUrl) { + try { + const inputUrl = regUrl.startsWith("http") ? regUrl : `https://${regUrl}`; + const inputHost = new URL(inputUrl).host.toLowerCase(); + const urlMatch = regList.find((r) => { + try { + return new URL(r.registry_url).host.toLowerCase() === inputHost; + } catch { + return false; + } + }); + if (urlMatch) return urlMatch; + } catch { + // invalid URL + } + } + + // 3. Check default npmjs match if no custom scope and no custom url + if (!detectedScope && !regUrl) { + const defaultMatch = regList.find( + (r) => !r.scope && r.registry_url.includes("registry.npmjs.org") + ); + if (defaultMatch) return defaultMatch; + } + + // 4. Fallback if single global/legacy token exists and neither custom scope nor custom url + if (!detectedScope && !regUrl && npmAuth.masked_token) { + return regList[0] || { + id: "default", + registry_url: npmAuth.default_registry || "https://registry.npmjs.org/", + masked_token: npmAuth.masked_token, + updated_at: "", + }; + } + + return null; + }, [npmAuth, detectedScope, enterpriseRegistryUrl]); + + const { data: officialCatalog = [], isLoading: officialLoading, error: officialError } = useSWR( + installModalOpen && catalogTab === "official" ? "plugin-catalog-official" : null, + () => pluginService.listCatalog("official") + ); + const { data: communityCatalog = [], isLoading: communityLoading, error: communityError } = useSWR( + installModalOpen && catalogTab === "community" ? "plugin-catalog-community" : null, + () => pluginService.listCatalog("community") + ); + + const [versionModalOpen, setVersionModalOpen] = useState(false); + const [selectedPlugin, setSelectedPlugin] = useState(null); + const [targetVersion, setTargetVersion] = useState(""); + const [updatingVersion, setUpdatingVersion] = useState(false); + + const [uninstallDialogOpen, setUninstallDialogOpen] = useState(false); + const [pluginToUninstall, setPluginToUninstall] = useState(null); + const [uninstalling, setUninstalling] = useState(false); + const [detailsPlugin, setDetailsPlugin] = useState(null); + + const handleInstall = async (plugin: PluginCatalogItem) => { + const version = selectedVersions[plugin.package] || "latest"; + setInstallingPkgs((prev) => new Set(prev).add(plugin.package)); + try { + await pluginService.install(plugin.package, version === "latest" ? undefined : version); + toast.success(`Plugin '${plugin.name}' installed successfully`); + mutate(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to install plugin"; + toast.error(message); + } finally { + setInstallingPkgs((prev) => { + const next = new Set(prev); + next.delete(plugin.package); + return next; + }); + } + }; + + const handleInstallCustom = async (e: React.FormEvent) => { + e.preventDefault(); + const pkgName = customPkg.trim(); + if (!pkgName) { + toast.error("Please enter a package name"); + return; + } + + const payload: InstallPluginPayload = { + package: pkgName, + version: customVersion.trim() || undefined, + }; + + if (isPrivate) { + const hasConfiguredAuth = matchedAuth && !overrideToken; + const token = hasConfiguredAuth ? undefined : npmToken.trim(); + if (!hasConfiguredAuth && !token) { + toast.error("Please enter an npm authentication token for this private package"); + return; + } + if (token || enterpriseRegistryUrl.trim() || detectedScope) { + payload.auth = { + token: token || undefined, + registry_url: enterpriseRegistryUrl.trim() || undefined, + scope: detectedScope, + save_token: saveToken, + }; + } + } + + setIsInstallingCustom(true); + setInstallingPkgs((prev) => new Set(prev).add(pkgName)); + try { + await pluginService.install(payload); + toast.success(`Plugin '${pkgName}' installed successfully!`); + mutate(); + if (payload.auth?.token && saveToken) { + mutateNpmAuth(); + } + setCustomPkg(""); + setCustomVersion(""); + setNpmToken(""); + setEnterpriseRegistryUrl(""); + setOverrideToken(false); + setIsPrivate(false); + setInstallModalOpen(false); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to install custom plugin"; + toast.error(message); + } finally { + setIsInstallingCustom(false); + setInstallingPkgs((prev) => { + const next = new Set(prev); + next.delete(pkgName); + return next; + }); + } + }; + + const catalog = catalogTab === "official" ? officialCatalog : communityCatalog; + const filteredCatalog = useMemo(() => { + const query = catalogSearch.trim().toLowerCase(); + if (!query) return catalog; + return catalog.filter((plugin) => + [plugin.name, plugin.package, plugin.description].some((value) => + value.toLowerCase().includes(query) + ) + ); + }, [catalog, catalogSearch]); + + const handleChangeVersion = async (e: React.FormEvent) => { + e.preventDefault(); + if (!selectedPlugin || !targetVersion.trim()) return; + + setUpdatingVersion(true); + try { + await pluginService.changeVersion(selectedPlugin.name, targetVersion.trim()); + toast.success(`Plugin '${selectedPlugin.name}' updated to ${targetVersion.trim()}`); + setVersionModalOpen(false); + setSelectedPlugin(null); + setTargetVersion(""); + mutate(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to change plugin version"; + toast.error(message); + } finally { + setUpdatingVersion(false); + } + }; + + const handleUninstall = async () => { + if (!pluginToUninstall) return; + + setUninstalling(true); + try { + await pluginService.uninstall(pluginToUninstall); + toast.success(`Plugin '${pluginToUninstall}' uninstalled successfully`); + setUninstallDialogOpen(false); + setPluginToUninstall(null); + mutate(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to uninstall plugin"; + toast.error(message); + } finally { + setUninstalling(false); + } + }; + + const plugins = useMemo(() => data?.plugins || [], [data?.plugins]); + const installedPackages = useMemo( + () => new Set(plugins.map((plugin) => plugin.name)), + [plugins] + ); + + return ( +
+ {/* Action Header */} +
+
+

Installed Plugin Packages

+

+ Manage npm provider packages installed into SimpleNS. +

+
+
+ + +
+
+ + {/* Plugins Grid */} + {isLoading ? ( +
+ {[1, 2, 3].map((i) => ( + + + + + + + + ))} +
+ ) : error ? ( + + + Failed to load installed plugins. + + + ) : plugins.length === 0 ? ( + + + +
+

No plugins installed

+

+ Install provider packages such as @simplens/nodemailer-gmail or @simplens/mock. +

+
+ +
+
+ ) : ( +
+ {plugins.map((plugin) => ( + + +
+
+

{plugin.name}

+

+ {plugin.manifest?.displayName || plugin.name} +

+
+ + v{plugin.version} + +
+ +
+ + Channel: {plugin.manifest?.channel || "generic"} + + + {plugin.status} + +
+ +

+ {plugin.manifest?.description || "No description provided."} +

+ +
+ + + +
+
+
+ ))} +
+ )} + + !open && setDetailsPlugin(null)}> + + + {detailsPlugin?.manifest?.displayName || detailsPlugin?.name} + {detailsPlugin?.name} + + {detailsPlugin && ( +
+

+ {detailsPlugin.manifest?.description || "No description provided."} +

+
+
+
Version
+
{detailsPlugin.version}
+
+
+
Channel
+
{detailsPlugin.manifest?.channel || "generic"}
+
+
+
Author
+
{detailsPlugin.manifest?.author || "Not specified"}
+
+
+
Status
+
{detailsPlugin.status}
+
+
+
+
Required credentials
+
+ {detailsPlugin.manifest?.requiredCredentials?.join(", ") || "None"} +
+
+
+
Optional configuration
+
+ {detailsPlugin.manifest?.optionalConfig?.join(", ") || "None"} +
+
+ {detailsPlugin.manifest?.homepage && ( + + Plugin homepage + + + )} +
+ )} +
+
+ + {/* Install Plugin Dialog */} + + + + Install Plugin Package + + Browse official and community provider packages or install custom plugins from npmjs. + + +
+ setCatalogTab(value as "official" | "community" | "custom")}> + + Official + Community + Custom Package + + + {catalogTab !== "custom" && ( +
+ + setCatalogSearch(event.target.value)} + /> +
+ )} + + + {officialLoading ? ( +
+ Loading plugins... +
+ ) : officialError ? ( +
+ Failed to load official plugins. +
+ ) : filteredCatalog.length === 0 ? ( +
+ No plugins match your search. +
+ ) : ( +
+ {filteredCatalog.map((plugin) => { + const versions = plugin.versions?.length ? plugin.versions : ["latest"]; + const selectedVersion = selectedVersions[plugin.package] || versions[0]; + const isInstalled = installedPackages.has(plugin.package); + const isInstalling = installingPkgs.has(plugin.package); + return ( +
+
+
{plugin.name}
+
{plugin.package}
+

{plugin.description}

+
+
+ + {isInstalled ? ( + + ) : ( + + )} +
+
+ ); + })} +
+ )} +
+ + + {communityLoading ? ( +
+ Loading plugins... +
+ ) : communityError ? ( +
+ Failed to load community plugins. +
+ ) : filteredCatalog.length === 0 ? ( +
+ {communityCatalog.length === 0 + ? "No community plugins are available yet." + : "No plugins match your search."} +
+ ) : ( +
+ {filteredCatalog.map((plugin) => { + const versions = plugin.versions?.length ? plugin.versions : ["latest"]; + const selectedVersion = selectedVersions[plugin.package] || versions[0]; + const isInstalled = installedPackages.has(plugin.package); + const isInstalling = installingPkgs.has(plugin.package); + return ( +
+
+
{plugin.name}
+
{plugin.package}
+

{plugin.description}

+
+
+ + {isInstalled ? ( + + ) : ( + + )} +
+
+ ); + })} +
+ )} +
+ + +
+
+
+ + Install from npmjs or Enterprise Registry +
+

+ Install any custom plugin implementing the SimpleNS Provider interface. For scoped packages (e.g. @myenterprise/provider), the scope is automatically detected. +

+
+ +
+
+ + setCustomPkg(e.target.value)} + required + disabled={isInstallingCustom} + /> +
+ +
+ + setCustomVersion(e.target.value)} + disabled={isInstallingCustom} + /> +
+
+ + {/* Private / Enterprise npm Package Toggle */} +
+
+
+
+ + Private / Enterprise Package +
+

+ Enable if this package requires an npm authentication token to download. +

+
+ +
+ + {isPrivate && ( +
+ {matchedAuth && !overrideToken ? ( +
+
+ +
+
+ Configured: Using saved credentials for {matchedAuth.scope || matchedAuth.registry_url} +
+
+ Active Token: {matchedAuth.masked_token} +
+
+
+ +
+ ) : ( +
+ {overrideToken && ( +
+ + Updating credentials for {detectedScope || "this registry"} + + +
+ )} + +
+
+ + + + + + +
+ + Token Requirements & Compatibility +
+
    +
  • + CI & Automation (No 2FA): SimpleNS runs installs in headless background processes. For npmjs.com, generate an Automation token (Classic token type "Automation") or a Granular Access Token configured with No 2FA prompt on install and Read-only permissions. +
  • +
  • + Scope Reusability: A single organization or user token covers all private packages under that scope (e.g. @my-org/*). You do not need a separate token for each package. +
  • +
  • + Enterprise Registries: For Verdaccio, Nexus, or Artifactory, enter an API key or service account token with read permissions. +
  • +
+
+
+
+ +
+ setNpmToken(e.target.value)} + required={isPrivate && (!matchedAuth || overrideToken)} + className="pr-10 font-mono text-xs" + disabled={isInstallingCustom} + /> + +
+
+ +
+ + setEnterpriseRegistryUrl(e.target.value)} + disabled={isInstallingCustom} + /> +
+ +
+ setSaveToken(e.target.checked)} + className="rounded text-primary h-4 w-4" + disabled={isInstallingCustom} + /> + +
+
+ )} +
+ )} +
+ +
+ +
+
+
+
+
+ + + +
+
+ + {/* Change Version Dialog */} + + +
+ + Change Plugin Version + + Upgrade or downgrade {selectedPlugin?.name} to a specific version. + + +
+
+ + setTargetVersion(e.target.value)} + required + /> +
+
+ + + + +
+
+
+ + {/* Uninstall Confirmation Dialog */} + + + + Uninstall Plugin + + Are you sure you want to uninstall {pluginToUninstall}? This will delete the package from + .plugins/node_modules. Any providers using this plugin and channel routings that reference those providers will also be deleted. + + + + Cancel + { + e.preventDefault(); + handleUninstall(); + }} + disabled={uninstalling} + className="bg-destructive text-destructive-foreground hover:bg-destructive/90" + > + {uninstalling ? "Uninstalling..." : "Uninstall"} + + + + +
+ ); +} diff --git a/dashboard/components/plugins/providers-tab.tsx b/dashboard/components/plugins/providers-tab.tsx new file mode 100644 index 0000000..abd7c3f --- /dev/null +++ b/dashboard/components/plugins/providers-tab.tsx @@ -0,0 +1,1016 @@ +"use client"; + +import { useState, useMemo } from "react"; +import useSWR from "swr"; +import { providerService, pluginService } from "@/lib/api-client"; +import type { ProviderDto, InstalledPlugin, ProviderRateLimitStatus, ProviderRateLimitListResponse } from "@/lib/types"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + Plus, + RefreshCw, + Trash2, + Edit2, + KeyRound, + TestTube2, + CheckCircle2, + AlertCircle, + Loader2, + Server, + Eye, + EyeOff, + Gauge, + RotateCcw, + AlertTriangle, +} from "lucide-react"; +import { toast } from "sonner"; +import { cn } from "@/lib/utils"; + +type AdditionalOption = { + id: string; + key: string; + value: string; +}; + +let additionalOptionId = 0; + +const createAdditionalOption = (key = "", value = ""): AdditionalOption => ({ + id: `additional-option-${additionalOptionId++}`, + key, + value, +}); + +function ProviderRateLimitRing({ + rateLimit, + onReset, + isResetting, +}: { + rateLimit?: ProviderRateLimitStatus; + onReset: () => void; + isResetting?: boolean; +}) { + const [open, setOpen] = useState(false); + + if (!rateLimit) { + return null; + } + + const { config, status } = rateLimit; + const maxTokens = config.max_tokens; + const remaining = status.remaining_tokens; + const used = status.used_tokens; + const isExhausted = status.is_exhausted; + + const remainingPct = maxTokens > 0 ? Math.min(100, Math.max(0, Math.round((remaining / maxTokens) * 100))) : 0; + const usedPct = 100 - remainingPct; + + let strokeColor = "text-emerald-500"; + let statusBadgeClass = "bg-emerald-500/15 text-emerald-600 dark:text-emerald-400 border-emerald-500/30"; + let statusText = "Healthy"; + + if (isExhausted) { + strokeColor = "text-rose-500"; + statusBadgeClass = "bg-rose-500/15 text-rose-600 dark:text-rose-400 border-rose-500/30"; + statusText = "Exhausted"; + } else if (remainingPct <= 30) { + strokeColor = "text-amber-500"; + statusBadgeClass = "bg-amber-500/15 text-amber-600 dark:text-amber-400 border-amber-500/30"; + statusText = "Low Capacity"; + } + + const size = 32; + const strokeWidth = 3; + const center = size / 2; + const radius = center - strokeWidth; + const circumference = 2 * Math.PI * radius; + const strokeDashoffset = circumference - (remainingPct / 100) * circumference; + + return ( + + + + + setOpen(true)} + onMouseLeave={() => setOpen(false)} + onClick={(e) => e.stopPropagation()} + > +
+
+ + Rate Limit Details +
+ + {statusText} + +
+ +
+
+ Tokens Remaining: + {remaining} / {maxTokens} +
+
+
+
+
+ Used: {used} ({usedPct}%) + Capacity: {maxTokens} +
+
+ +
+
+ Refill Cadence: + +{config.refill_rate} / {config.refill_interval} +
+ {status.is_exhausted ? ( +
+ + Next token in {(status.resets_in_ms / 1000).toFixed(1)}s +
+ ) : remaining < maxTokens ? ( +
+ Full bucket in: + {(status.full_refill_in_ms / 1000).toFixed(1)}s +
+ ) : ( +
+ Bucket is at full capacity +
+ )} + {status.exhausted_count > 0 && ( +
+ Exhaustion events: + {status.exhausted_count} +
+ )} +
+ +
+ +
+ + + ); +} + +export function ProvidersTab() { + const { + data: providersData, + isLoading: providersLoading, + mutate: mutateProviders, + } = useSWR<{ providers: ProviderDto[] }>("/api/providers", () => providerService.list()); + + const { data: pluginsData } = useSWR<{ plugins: InstalledPlugin[] }>( + "/api/plugins/installed", + () => pluginService.listInstalled() + ); + + const { + data: rateLimitsData, + mutate: mutateRateLimits, + } = useSWR( + "/api/providers/rate-limits", + () => providerService.getRateLimits(), + { refreshInterval: 5000, revalidateOnFocus: true } + ); + + const rateLimitsMap = useMemo(() => { + const map = new Map(); + if (rateLimitsData?.providers) { + for (const rl of rateLimitsData.providers) { + map.set(rl.provider_id, rl); + } + } + return map; + }, [rateLimitsData]); + + const [resettingProviderId, setResettingProviderId] = useState(null); + + const handleResetRateLimit = async (id: string) => { + try { + setResettingProviderId(id); + await providerService.resetRateLimit(id); + toast.success(`Rate limit for provider '${id}' reset successfully`); + await mutateRateLimits(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to reset rate limit"; + toast.error(message); + } finally { + setResettingProviderId(null); + } + }; + + const [modalOpen, setModalOpen] = useState(false); + const [editingProvider, setEditingProvider] = useState(null); + + // Form State + const [providerId, setProviderId] = useState(""); + const [pluginName, setPluginName] = useState(""); + const [priority, setPriority] = useState(0); + const [rateLimit, setRateLimit] = useState({ + maxTokens: "", + refillRate: "", + refillInterval: "second", + }); + const [additionalOptions, setAdditionalOptions] = useState([]); + const [enabled, setEnabled] = useState(true); + const [credentials, setCredentials] = useState>({}); + const [visibleCredentials, setVisibleCredentials] = useState>({}); + const [loadingCredentialVisibility, setLoadingCredentialVisibility] = useState>({}); + const [saving, setSaving] = useState(false); + + // Test Connection State + const [testing, setTesting] = useState(false); + const [testResult, setTestResult] = useState<{ success: boolean; message: string } | null>(null); + + // Delete State + const [deleteDialogOpen, setDeleteDialogOpen] = useState(false); + const [providerToDelete, setProviderToDelete] = useState(null); + const [deleting, setDeleting] = useState(false); + + const installedPlugins = pluginsData?.plugins || []; + const selectedPlugin = installedPlugins.find((p) => p.name === pluginName); + + const resetForm = () => { + setEditingProvider(null); + setProviderId(""); + setPluginName(installedPlugins[0]?.name || ""); + setPriority(0); + setRateLimit({ maxTokens: "", refillRate: "", refillInterval: "second" }); + setAdditionalOptions([]); + setEnabled(true); + setCredentials({}); + setVisibleCredentials({}); + setLoadingCredentialVisibility({}); + setTestResult(null); + }; + + const handleOpenCreate = () => { + resetForm(); + if (installedPlugins.length > 0) { + setPluginName(installedPlugins[0].name); + } + setModalOpen(true); + }; + + const handleOpenEdit = async (provider: ProviderDto) => { + setEditingProvider(provider); + setProviderId(provider.id); + setPluginName(provider.plugin_name); + setPriority(provider.options?.priority ?? 0); + setRateLimit({ + maxTokens: provider.options?.rateLimit?.maxTokens?.toString() || "", + refillRate: provider.options?.rateLimit?.refillRate?.toString() || "", + refillInterval: provider.options?.rateLimit?.refillInterval || "second", + }); + setAdditionalOptions( + Object.entries(provider.options || {}) + .filter(([key]) => key !== "priority" && key !== "rateLimit") + .map(([key, value]) => ({ + ...createAdditionalOption( + key, + typeof value === "string" ? value : JSON.stringify(value), + ), + })) + ); + setEnabled(provider.enabled); + setCredentials({}); + setVisibleCredentials({}); + setLoadingCredentialVisibility({}); + setTestResult(null); + setModalOpen(true); + }; + + const buildProviderOptions = (): Record => { + const options: Record = { priority }; + const configuredRateLimit: Record = {}; + + if (rateLimit.maxTokens.trim() !== "") { + configuredRateLimit.maxTokens = parseInt(rateLimit.maxTokens, 10); + } + if (rateLimit.refillRate.trim() !== "") { + configuredRateLimit.refillRate = parseInt(rateLimit.refillRate, 10); + } + if (rateLimit.refillInterval) { + configuredRateLimit.refillInterval = rateLimit.refillInterval; + } + if (Object.keys(configuredRateLimit).length > 0) { + options.rateLimit = configuredRateLimit; + } + + for (const option of additionalOptions) { + const key = option.key.trim(); + if (!key || key === "priority" || key === "rateLimit") continue; + const value = option.value.trim(); + options[key] = /^-?\d+$/.test(value) ? parseInt(value, 10) : option.value; + } + + return options; + }; + + const toggleCredentialVisibility = async (field: string): Promise => { + if (visibleCredentials[field]) { + setVisibleCredentials({ ...visibleCredentials, [field]: false }); + return; + } + + if (!editingProvider) return; + + if (!credentials[field]) { + setLoadingCredentialVisibility({ ...loadingCredentialVisibility, [field]: true }); + try { + const result = await providerService.get(editingProvider.id, true); + const decryptedCredentials = result.provider.decrypted_credentials || {}; + setCredentials({ ...credentials, ...decryptedCredentials }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : `Failed to reveal ${field}`; + toast.error(message); + return; + } finally { + setLoadingCredentialVisibility({ ...loadingCredentialVisibility, [field]: false }); + } + } + + setVisibleCredentials({ ...visibleCredentials, [field]: true }); + }; + + const handleTestConnection = async () => { + setTesting(true); + setTestResult(null); + try { + const result = await providerService.test({ + provider_id: editingProvider ? editingProvider.id : undefined, + plugin_name: pluginName, + credentials: Object.keys(credentials).length > 0 ? credentials : undefined, + }); + setTestResult(result); + if (result.success) { + toast.success("Health check succeeded!"); + } else { + toast.error(result.message || "Health check failed"); + } + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Connection test failed"; + setTestResult({ success: false, message }); + toast.error(message); + } finally { + setTesting(false); + } + }; + + const handleSave = async (e: React.FormEvent) => { + e.preventDefault(); + setSaving(true); + try { + if (editingProvider) { + await providerService.update(editingProvider.id, { + options: buildProviderOptions(), + enabled, + credentials: Object.keys(credentials).length > 0 ? credentials : undefined, + }); + toast.success(`Provider '${editingProvider.id}' updated`); + } else { + if (!providerId.trim()) { + toast.error("Provider ID is required"); + return; + } + await providerService.create({ + id: providerId.trim(), + plugin_name: pluginName, + credentials, + options: buildProviderOptions(), + enabled, + }); + toast.success(`Provider '${providerId}' created`); + } + setModalOpen(false); + resetForm(); + mutateProviders(); + mutateRateLimits(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to save provider"; + toast.error(message); + } finally { + setSaving(false); + } + }; + + const handleDelete = async () => { + if (!providerToDelete) return; + setDeleting(true); + try { + await providerService.delete(providerToDelete); + toast.success(`Provider '${providerToDelete}' deleted`); + setDeleteDialogOpen(false); + setProviderToDelete(null); + mutateProviders(); + mutateRateLimits(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to delete provider"; + toast.error(message); + } finally { + setDeleting(false); + } + }; + + const handleToggleEnabled = async (provider: ProviderDto, newEnabled: boolean) => { + try { + await providerService.update(provider.id, { enabled: newEnabled }); + toast.success(`Provider '${provider.id}' ${newEnabled ? "enabled" : "disabled"}`); + mutateProviders(); + mutateRateLimits(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to update provider status"; + toast.error(message); + } + }; + + const providers = providersData?.providers || []; + + return ( +
+ {/* Header */} +
+
+

Configured Providers

+

+ Configure provider instances with envelope-encrypted credentials and priorities. +

+
+
+ + +
+
+ + {installedPlugins.length === 0 && ( + + + + + No plugins are installed yet. Please switch to the Installed Plugins{" "} + tab and install a plugin package first before creating providers. + + + + )} + + {/* Providers Grid */} + {providersLoading ? ( +
+ {[1, 2, 3].map((i) => ( + + + + + + + + ))} +
+ ) : providers.length === 0 ? ( + + + +
+

No providers configured

+

+ Create a provider instance for your installed plugins. +

+
+ +
+
+ ) : ( +
+ {providers.map((p) => ( + + +
+
+

{p.id}

+

+ {p.plugin_name} +

+
+
+ handleResetRateLimit(p.id)} + isResetting={resettingProviderId === p.id} + /> + + {p.enabled ? "Enabled" : "Disabled"} + +
+
+ +
+ Channel: {p.channel} + Priority: {p.options?.priority || 0} +
+ + Encrypted (RSA-OAEP) +
+
+ +
+
+ handleToggleEnabled(p, checked)} + /> + Active +
+ +
+ + +
+
+
+
+ ))} +
+ )} + + {/* Add/Edit Provider Modal */} + + +
+ + {editingProvider ? "Edit Provider" : "Configure Provider"} + + {editingProvider + ? `Update settings or rotate encrypted credentials for ${editingProvider.id}.` + : "Create an active provider instance backed by an installed plugin."} + + + +
+ {!editingProvider ? ( + <> +
+ + +
+ +
+ + setProviderId(e.target.value)} + required + /> +
+ + ) : ( +
+
+ Provider ID: {editingProvider.id} +
+
+ Plugin: {editingProvider.plugin_name} +
+
+ Channel: {editingProvider.channel} +
+
+ )} + +
+
+ + setPriority(parseInt(e.target.value, 10) || 0)} + /> +
+
+
+ + +
+
+
+ +
+ +
+
+ + setRateLimit({ ...rateLimit, maxTokens: e.target.value })} + placeholder="100" + /> +
+
+ + setRateLimit({ ...rateLimit, refillRate: e.target.value })} + placeholder="10" + /> +
+
+ + +
+
+
+ +
+
+ + +
+ {additionalOptions.map((option, index) => ( +
+ { + const next = [...additionalOptions]; + next[index] = { ...next[index], key: e.target.value }; + setAdditionalOptions(next); + }} + /> + { + const next = [...additionalOptions]; + next[index] = { ...next[index], value: e.target.value }; + setAdditionalOptions(next); + }} + /> + +
+ ))} +
+ + {/* Dynamic Credential Fields from Plugin Manifest */} +
+
+ + + + Envelope Encrypted + +
+ {editingProvider && ( +

+ Leave credential fields empty to test with the encrypted credentials already stored for this provider. +

+ )} + + {selectedPlugin?.manifest?.requiredCredentials && + selectedPlugin.manifest.requiredCredentials.length > 0 ? ( + selectedPlugin.manifest.requiredCredentials.map((field) => ( +
+ +
+ + setCredentials({ ...credentials, [field]: e.target.value }) + } + required={!editingProvider} + /> + {editingProvider && ( + + )} +
+
+ )) + ) : ( +

+ This plugin does not specify required credentials. +

+ )} +
+ + {/* Live Test Connection Result */} + {testResult && ( +
+ {testResult.success ? ( + + ) : ( + + )} + {testResult.message} +
+ )} +
+ + + {editingProvider && ( + + )} + +
+ + +
+
+
+
+
+ + {/* Delete Confirmation Dialog */} + + + + Delete Provider + + Are you sure you want to delete {providerToDelete}? This will unregister the instance and remove it from any channel routing. If it is a channel's primary provider, the first fallback will become primary. + + + + Cancel + { + e.preventDefault(); + handleDelete(); + }} + disabled={deleting} + className="bg-destructive text-destructive-foreground hover:bg-destructive/90" + > + {deleting ? "Deleting..." : "Delete Provider"} + + + + +
+ ); +} diff --git a/dashboard/components/send/batch-notification-form.tsx b/dashboard/components/send/batch-notification-form.tsx index 7c56669..20cdd58 100644 --- a/dashboard/components/send/batch-notification-form.tsx +++ b/dashboard/components/send/batch-notification-form.tsx @@ -16,6 +16,7 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { pluginService, templateService, notificationService } from "@/lib/api-client"; +import { withBasePath } from "@/lib/utils"; // Generate a UUIDv4 function generateUUID(): string { @@ -400,8 +401,9 @@ export function BatchNotificationForm({ onSuccess }: BatchNotificationFormProps) client_id: clientId, channel: selectedChannels, recipients: processedRecipients, - //hard-coded the webhook url as the simplest solution. This url may not be correct all the times. Nop issues, the worker just fire and forgets while sending webhook, it does not retires - webhook_url: "http://localhost:3002/api/webhook" + webhook_url: typeof window !== "undefined" + ? `${window.location.origin}${withBasePath("/api/webhook")}` + : "http://localhost:3000/api/webhook", }; const hasAnyTemplate = selectedChannels.some( diff --git a/dashboard/components/send/single-notification-form.tsx b/dashboard/components/send/single-notification-form.tsx index 1346c97..f00130c 100644 --- a/dashboard/components/send/single-notification-form.tsx +++ b/dashboard/components/send/single-notification-form.tsx @@ -16,6 +16,7 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { pluginService, templateService, notificationService } from "@/lib/api-client"; +import { withBasePath } from "@/lib/utils"; // Generate a UUIDv4 function generateUUID(): string { @@ -324,8 +325,9 @@ export function SingleNotificationForm({ onSuccess }: SingleNotificationFormProp client_id: clientId, channel: selectedChannels, recipient: { ...recipientData }, - //hard-coded the webhook url as the simplest solution. This url may not be correct all the times. Nop issues, the worker just fire and forgets while sending webhook, it does not retires - webhook_url: "http://localhost:3002/api/webhook" + webhook_url: typeof window !== "undefined" + ? `${window.location.origin}${withBasePath("/api/webhook")}` + : "http://localhost:3000/api/webhook", }; const hasAnyTemplate = selectedChannels.some( @@ -437,7 +439,7 @@ export function SingleNotificationForm({ onSuccess }: SingleNotificationFormProp No Plugins Found - No notification plugins are configured. Please install and configure plugins in simplens.config.yaml. + No notification plugins are configured. Please install and configure plugins. ); diff --git a/dashboard/lib/api-client.ts b/dashboard/lib/api-client.ts index 7d6fa92..1f443cb 100644 --- a/dashboard/lib/api-client.ts +++ b/dashboard/lib/api-client.ts @@ -19,6 +19,20 @@ import { AdminChannel, AdminChannelFormData, AdminChannelProviderMeta, + InstalledPlugin, + PluginCatalogItem, + NpmAuthStatus, + InstallPluginPayload, + ProviderDto, + ChannelRoutingDto, + OperationalSettings, + AdminAuthStatus, + ProviderRateLimitStatus, + ProviderRateLimitListResponse, + ApiKey, + CreateApiKeyPayload, + CreateApiKeyResponse, + ApiKeyUsageDetailResponse, } from './types'; export class ApiError extends Error { @@ -37,6 +51,7 @@ export const apiClient = axios.create({ headers: { 'Content-Type': 'application/json', }, + withCredentials: true, }); // Dynamic Base URL Resolution @@ -69,11 +84,25 @@ apiClient.interceptors.response.use( // Service Modules export const authService = { - login: (payload: Record): Promise<{ success: boolean; redirectUrl: string }> => + signup: (payload: { username: string; password: string }): Promise<{ success: boolean; message?: string; redirectUrl?: string }> => + apiClient.post('/api/auth/setup', payload), + setup: (payload: { username: string; password: string }): Promise<{ success: boolean; message?: string; redirectUrl?: string }> => + apiClient.post('/api/auth/setup', payload), + login: (payload: Record): Promise<{ success: boolean; isValid?: boolean; message?: string; redirectUrl?: string }> => apiClient.post('/api/auth/login', payload), - logout: (): Promise => apiClient.post('/api/auth/logout'), - getSession: (): Promise<{ authenticated: boolean; user?: { id: string; username: string } }> => - apiClient.get('/api/auth/session'), + logout: (): Promise<{ success: boolean; message?: string }> => + apiClient.post('/api/auth/logout'), + getAuthStatus: (): Promise => + apiClient.get('/api/auth/status'), +}; + +export const settingsService = { + get: (): Promise<{ success: boolean; settings: OperationalSettings }> => + apiClient.get('/api/settings'), + update: (payload: Partial): Promise<{ success: boolean; message: string; settings: OperationalSettings }> => + apiClient.put('/api/settings', payload), + reset: (): Promise<{ success: boolean; message: string; settings: OperationalSettings }> => + apiClient.post('/api/settings/reset'), }; export const notificationService = { @@ -127,7 +156,71 @@ export const alertService = { }; export const pluginService = { + listCatalog: (category: 'official' | 'community'): Promise => + apiClient.get(`/api/plugins/catalog/${category}`), getMetadata: (): Promise => apiClient.get('/api/plugins'), + listInstalled: (): Promise<{ plugins: InstalledPlugin[] }> => apiClient.get('/api/plugins/installed'), + install: (payload: string | InstallPluginPayload, version?: string): Promise<{ message: string; plugin: InstalledPlugin }> => { + const body = typeof payload === 'string' ? { package: payload, version } : payload; + return apiClient.post('/api/plugins/install', body); + }, + changeVersion: (packageName: string, version: string): Promise<{ message: string; plugin: InstalledPlugin }> => + apiClient.put('/api/plugins/version', { package: packageName, version }), + uninstall: (packageName: string): Promise<{ message: string }> => + apiClient.delete(`/api/plugins/${encodeURIComponent(packageName)}`), + getNpmAuth: (): Promise => apiClient.get('/api/plugins/npm-auth'), + saveNpmAuth: (payload: { token: string; registry_url?: string; scope?: string }): Promise<{ message: string; status: NpmAuthStatus }> => + apiClient.post('/api/plugins/npm-auth', payload), + deleteNpmAuth: (idOrScope?: string): Promise<{ message: string }> => + apiClient.delete('/api/plugins/npm-auth', { params: idOrScope ? { id: idOrScope } : undefined }), +}; + +export const providerService = { + list: (): Promise<{ providers: ProviderDto[] }> => apiClient.get('/api/providers'), + get: (id: string, includeDecrypted = false): Promise<{ provider: ProviderDto }> => + apiClient.get(`/api/providers/${id}`, { params: { include_decrypted: includeDecrypted } }), + create: (payload: { + id: string; + plugin_name: string; + credentials: Record; + options?: Record & { priority?: number }; + enabled?: boolean; + }): Promise<{ message: string; provider: ProviderDto }> => + apiClient.post('/api/providers', payload), + update: (id: string, payload: { + credentials?: Record; + options?: Record & { priority?: number }; + enabled?: boolean; + }): Promise<{ message: string; provider: ProviderDto }> => + apiClient.put(`/api/providers/${id}`, payload), + delete: (id: string): Promise<{ message: string }> => + apiClient.delete(`/api/providers/${id}`), + test: (payload: { + provider_id?: string; + plugin_name?: string; + credentials?: Record; + options?: Record; + }): Promise<{ success: boolean; message: string }> => + apiClient.post('/api/providers/test', payload), + getRateLimits: (): Promise => + apiClient.get('/api/providers/rate-limits'), + getRateLimit: (id: string): Promise<{ rate_limit: ProviderRateLimitStatus }> => + apiClient.get(`/api/providers/${id}/rate-limit`), + resetRateLimit: (id: string): Promise<{ message: string }> => + apiClient.post(`/api/providers/${id}/rate-limit/reset`), +}; + +export const channelRoutingService = { + list: (): Promise<{ routings: ChannelRoutingDto[] }> => apiClient.get('/api/channels/routing'), + get: (channel: string): Promise<{ routing: ChannelRoutingDto }> => apiClient.get(`/api/channels/routing/${channel}`), + set: (channel: string, payload: { + default_provider_id: string; + fallback_provider_ids?: string[]; + partitions?: number; + }): Promise<{ message: string; routing: ChannelRoutingDto }> => + apiClient.put(`/api/channels/routing/${channel}`, payload), + delete: (channel: string): Promise<{ message: string }> => + apiClient.delete(`/api/channels/routing/${channel}`), }; export const adminChannelService = { @@ -150,3 +243,15 @@ export const dashboardService = { getTrends: (period?: string): Promise => apiClient.get('/api/dashboard/trends', { params: { period } }), }; + +export const apiKeyService = { + list: (): Promise<{ keys: ApiKey[] }> => apiClient.get('/api/keys'), + create: (payload: CreateApiKeyPayload): Promise => + apiClient.post('/api/keys', payload), + getUsage: (id: string): Promise => + apiClient.get(`/api/keys/${id}`), + revoke: (id: string): Promise<{ success: boolean; message: string; key: ApiKey }> => + apiClient.post(`/api/keys/${id}/revoke`), + delete: (id: string): Promise<{ success: boolean; message: string }> => + apiClient.delete(`/api/keys/${id}`), +}; diff --git a/dashboard/lib/api-config.ts b/dashboard/lib/api-config.ts index c777abd..e985e51 100644 --- a/dashboard/lib/api-config.ts +++ b/dashboard/lib/api-config.ts @@ -1,7 +1,12 @@ /** * API Configuration - * Centralized configuration for API endpoints + * Centralized configuration for backend API base URL resolution */ -export const API_BASE_URL = process.env.API_BASE_URL || 'http://localhost:3000'; -export const NS_API_KEY = process.env.NS_API_KEY || ""; \ No newline at end of file +const rawBaseUrl = (process.env.API_BASE_URL || 'http://localhost:3000').trim().replace(/\/+$/, ''); +const rawBasePath = (process.env.BASE_PATH || '').trim().replace(/\/+$/, ''); +const normalizedBasePath = rawBasePath.startsWith('/') ? rawBasePath : rawBasePath ? `/${rawBasePath}` : ''; + +export const API_BASE_URL = normalizedBasePath && !rawBaseUrl.endsWith(normalizedBasePath) + ? `${rawBaseUrl}${normalizedBasePath}` + : rawBaseUrl; diff --git a/dashboard/lib/session.ts b/dashboard/lib/session.ts deleted file mode 100644 index 6f6fd3e..0000000 --- a/dashboard/lib/session.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { cookies } from "next/headers"; -import { createHmac } from "crypto"; - -const SESSION_COOKIE_NAME = "simplens_session"; -const SESSION_EXPIRY_DAYS = 7; - -interface SessionPayload { - userId: string; - username: string; - createdAt: number; - expiresAt: number; -} - -/** - * Get the auth secret from environment, throws if not configured - */ -function getAuthSecret(): string { - const secret = process.env.AUTH_SECRET; - if (!secret) { - throw new Error("AUTH_SECRET environment variable is not configured"); - } - return secret; -} - -/** - * Sign a payload using HMAC-SHA256 - */ -function signPayload(payload: string): string { - const hmac = createHmac("sha256", getAuthSecret()); - hmac.update(payload); - return hmac.digest("hex"); -} - -/** - * Verify a signed token and return the payload if valid - */ -function verifyToken(token: string): SessionPayload | null { - try { - const [payloadBase64, signature] = token.split("."); - if (!payloadBase64 || !signature) { - return null; - } - - const expectedSignature = signPayload(payloadBase64); - if (signature !== expectedSignature) { - return null; - } - - const payload = JSON.parse( - Buffer.from(payloadBase64, "base64").toString("utf-8") - ) as SessionPayload; - - // Check if session has expired - if (Date.now() > payload.expiresAt) { - return null; - } - - return payload; - } catch { - return null; - } -} - -/** - * Create a signed session token - */ -function createToken(payload: SessionPayload): string { - const payloadBase64 = Buffer.from(JSON.stringify(payload)).toString("base64"); - const signature = signPayload(payloadBase64); - return `${payloadBase64}.${signature}`; -} - -/** - * Create a session for the authenticated user - */ -export async function createSession(userId: string, username: string): Promise { - const now = Date.now(); - const expiresAt = now + SESSION_EXPIRY_DAYS * 24 * 60 * 60 * 1000; - - const payload: SessionPayload = { - userId, - username, - createdAt: now, - expiresAt, - }; - - const token = createToken(payload); - - const cookieStore = await cookies(); - cookieStore.set(SESSION_COOKIE_NAME, token, { - httpOnly: true, - secure: process.env.HTTPS_COOKIE !== "false", - sameSite: "lax", - path: "/", - maxAge: SESSION_EXPIRY_DAYS * 24 * 60 * 60, - }); -} - -/** - * Validate the current session and return user info if valid - */ -export async function validateSession(): Promise<{ - isValid: boolean; - user?: { id: string; username: string }; -}> { - try { - const cookieStore = await cookies(); - const token = cookieStore.get(SESSION_COOKIE_NAME)?.value; - - if (!token) { - return { isValid: false }; - } - - const payload = verifyToken(token); - if (!payload) { - return { isValid: false }; - } - - return { - isValid: true, - user: { - id: payload.userId, - username: payload.username, - }, - }; - } catch { - return { isValid: false }; - } -} - -/** - * Destroy the current session - */ -export async function destroySession(): Promise { - const cookieStore = await cookies(); - cookieStore.delete(SESSION_COOKIE_NAME); -} - -/** - * Validate credentials against environment variables - */ -export function validateCredentials( - username: string, - password: string -): { isValid: boolean; userId?: string } { - const adminUsername = process.env.ADMIN_USERNAME; - const adminPassword = process.env.ADMIN_PASSWORD; - - if (!adminUsername || !adminPassword) { - console.error("ADMIN_USERNAME or ADMIN_PASSWORD not configured in .env"); - return { isValid: false }; - } - - if (username === adminUsername && password === adminPassword) { - return { isValid: true, userId: "admin-1" }; - } - - return { isValid: false }; -} diff --git a/dashboard/lib/types.ts b/dashboard/lib/types.ts index d986947..3cc5ed5 100644 --- a/dashboard/lib/types.ts +++ b/dashboard/lib/types.ts @@ -22,6 +22,17 @@ export interface Recipient { // Generic content - structure depends on channel export type NotificationContent = Record; +export interface ProviderAttempt { + provider: string; + status: 'delivered' | 'failed' | 'rate_limited'; + error_code?: string; + error_message?: string; + retryable?: boolean; + attempted_at: Date | string; + duration_ms?: number; + response?: unknown; +} + export interface Notification { _id: string; request_id: string; @@ -29,6 +40,7 @@ export interface Notification { client_name?: string; channel: Channel; provider?: string; // Provider ID used for this notification + provider_history?: ProviderAttempt[]; recipient: Recipient; content: NotificationContent; variables?: Record; @@ -216,3 +228,222 @@ export interface AdminChannelProviderMeta { displayName: string; credentialFields: AdminChannelProviderField[]; } + +// ============================================================================ +// DYNAMIC PLUGIN & PROVIDER MANAGEMENT TYPES +// ============================================================================ + +export interface ProviderManifest { + name: string; + displayName: string; + version: string; + channel: string; + description: string; + author?: string; + homepage?: string; + requiredCredentials: string[]; + optionalConfig?: string[]; +} + +export interface InstalledPlugin { + _id?: string; + name: string; + version: string; + status: 'installed' | 'installing' | 'failed'; + error?: string; + manifest: ProviderManifest; + created_at?: string; + updated_at?: string; +} + +export interface PluginCatalogItem { + name: string; + package: string; + description: string; + versions?: string[]; +} + +export interface NpmRegistryStatus { + id: string; + scope?: string; + registry_url: string; + masked_token: string; + updated_at: string; +} + +export interface NpmAuthStatus { + is_configured: boolean; + registries?: NpmRegistryStatus[]; + default_registry?: string; + masked_token?: string; +} + +export interface InstallPluginPayload { + package: string; + version?: string; + auth?: { + token?: string; + registry_url?: string; + scope?: string; + save_token?: boolean; + }; +} + +export interface ProviderDto { + _id?: string; + id: string; + plugin_name: string; + channel: string; + enabled: boolean; + options?: { + priority?: number; + rateLimit?: { + maxTokens?: number; + refillRate?: number; + refillInterval?: "second" | "minute" | "hour" | "day"; + }; + [key: string]: unknown; + }; + credentials_configured: boolean; + decrypted_credentials?: Record; + created_at?: string; + updated_at?: string; +} + +export interface ChannelRoutingDto { + _id?: string; + channel: string; + default_provider_id: string; + fallback_provider_ids: string[]; + partitions: number; + created_at?: string; + updated_at?: string; +} + +export interface OperationalSettings { + api: { + max_batch_req_limit: number; + }; + worker: { + outbox_poll_interval_ms: number; + outbox_cleanup_interval_ms: number; + outbox_batch_size: number; + outbox_retention_ms: number; + outbox_claim_timeout_ms: number; + }; + retry: { + max_retry_count: number; + idempotency_ttl_seconds: number; + processing_ttl_seconds: number; + rate_limit_retry_delay_ms: number; + }; + delayed: { + delayed_poll_interval_ms: number; + delayed_batch_size: number; + max_poller_retries: number; + }; + recovery: { + recovery_poll_interval_ms: number; + processing_stuck_threshold_ms: number; + pending_stuck_threshold_ms: number; + recovery_batch_size: number; + recovery_claim_timeout_ms: number; + cleanup_resolved_alerts_retention_ms: number; + cleanup_processed_status_outbox_retention_ms: number; + }; + logging: { + log_level: 'debug' | 'info' | 'warn' | 'error'; + log_to_file: boolean; + }; +} + +export interface AdminAuthStatus { + isConfigured: boolean; +} + +export interface ProviderRateLimitConfigDetails { + max_tokens: number; + refill_rate: number; + refill_interval: "second" | "minute" | "hour" | "day"; + normalized_refill_rate: number; +} + +export interface ProviderRateLimitRealtimeStatus { + remaining_tokens: number; + used_tokens: number; + usage_percentage: number; + is_exhausted: boolean; + resets_in_ms: number; + full_refill_in_ms: number; + last_refill_at: string | null; + last_exhausted_at: string | null; + exhausted_count: number; +} + +export interface ProviderRateLimitStatus { + provider_id: string; + channel: string; + plugin_name: string; + display_name?: string; + enabled: boolean; + config: ProviderRateLimitConfigDetails; + status: ProviderRateLimitRealtimeStatus; +} + +export interface ProviderRateLimitSummary { + total_providers: number; + exhausted_providers: number; + healthy_providers: number; +} + +export interface ProviderRateLimitListResponse { + providers: ProviderRateLimitStatus[]; + summary: ProviderRateLimitSummary; +} + +// ============================================================================ +// API KEY MANAGEMENT TYPES +// ============================================================================ + +export interface ApiKeyUsage { + total_requests: number; + total_notifications: number; + by_channel: Record; + last_used_at?: string | null; +} + +export type ApiKeyType = 'notification' | 'admin'; + +export interface ApiKey { + key_id: string; + name: string; + type: ApiKeyType; + key_prefix: string; + status: 'active' | 'revoked'; + expires_at?: string | null; + usage: ApiKeyUsage; + created_at: string; +} + +export interface CreateApiKeyPayload { + name: string; + type?: ApiKeyType; + expires_at?: string; +} + +export interface CreateApiKeyResponse { + key: ApiKey; + raw_key: string; +} + +export interface ApiKeyUsageDetailResponse { + key: ApiKey; + status_breakdown: { + pending: number; + processing: number; + delivered: number; + failed: number; + }; +} + + diff --git a/dashboard/middleware.ts b/dashboard/middleware.ts index 4098b46..ae9f026 100644 --- a/dashboard/middleware.ts +++ b/dashboard/middleware.ts @@ -1,23 +1,15 @@ import { NextResponse } from "next/server"; import type { NextRequest } from "next/server"; -import { API_BASE_URL, NS_API_KEY } from "./lib/api-config"; +import { API_BASE_URL } from "./lib/api-config"; const basePath = process.env.BASE_PATH || ""; const SESSION_COOKIE_NAME = "simplens_session"; -const localApiPaths = [ - "/api/auth/", - "/api/runtime-config", - "/api/webhook" -]; - -function isLocalApiRoute(pathname: string): boolean { - return localApiPaths.some((p) => pathname.startsWith(p)); -} - -function proxyToBackend(request: NextRequest, pathname: string) { +function proxyToBackend(request: NextRequest, pathname: string, jwtToken?: string) { const requestHeaders = new Headers(request.headers); - requestHeaders.set("Authorization", `Bearer ${NS_API_KEY}`); + if (jwtToken) { + requestHeaders.set("Authorization", `Bearer ${jwtToken}`); + } const backendUrl = new URL(`${API_BASE_URL}${pathname}${request.nextUrl.search}`); return NextResponse.rewrite(backendUrl, { @@ -30,9 +22,11 @@ function proxyToBackend(request: NextRequest, pathname: string) { // Routes that don't require authentication const publicRoutes = [ "/login", + "/setup", "/api/auth/login", + "/api/auth/setup", + "/api/auth/status", "/api/auth/logout", - "/api/auth/session", "/api/webhook", ]; @@ -47,157 +41,25 @@ const staticPatterns = [ ".ico", ]; -interface SessionPayload { - userId: string; - username: string; - createdAt: number; - expiresAt: number; -} - -function getAuthSecret(): string { - const secret = process.env.AUTH_SECRET; - if (!secret) { - throw new Error("AUTH_SECRET environment variable is not configured"); - } - return secret; -} - -// Web Crypto API compatible HMAC-SHA256 -async function signPayload(payload: string): Promise { - const encoder = new TextEncoder(); - const secretKey = await crypto.subtle.importKey( - "raw", - encoder.encode(getAuthSecret()), - { name: "HMAC", hash: "SHA-256" }, - false, - ["sign"] - ); - const signature = await crypto.subtle.sign( - "HMAC", - secretKey, - encoder.encode(payload) - ); - return Array.from(new Uint8Array(signature)) - .map((b) => b.toString(16).padStart(2, "0")) - .join(""); -} - -async function verifyToken(token: string): Promise { - try { - const [payloadBase64, signature] = token.split("."); - if (!payloadBase64 || !signature) { - return null; - } - - const expectedSignature = await signPayload(payloadBase64); - if (signature !== expectedSignature) { - return null; - } - - const payload = JSON.parse(atob(payloadBase64)) as SessionPayload; - - // Check if session has expired - if (Date.now() > payload.expiresAt) { - return null; - } - - return payload; - } catch { - return null; - } -} - -async function validateSessionFromRequest(request: NextRequest): Promise<{ +function validateSessionFromRequest(request: NextRequest): { isValid: boolean; - user?: { id: string; username: string }; -}> { - try { - const token = request.cookies.get(SESSION_COOKIE_NAME)?.value; - - if (!token) { - return { isValid: false }; - } - - const payload = await verifyToken(token); - if (!payload) { - return { isValid: false }; - } - - return { - isValid: true, - user: { - id: payload.userId, - username: payload.username, - }, - }; - } catch { - return { isValid: false }; - } -} - -/** - * Validate an API key from the Authorization: Bearer header. - * Uses timing-safe comparison via HMAC to prevent timing attacks. - */ -async function validateApiKeyFromRequest(request: NextRequest): Promise<{ isValid: boolean }> { - try { - if (!NS_API_KEY) { - return { isValid: false }; - } - - const authHeader = request.headers.get("authorization"); - if (!authHeader || !authHeader.startsWith("Bearer ")) { - return { isValid: false }; - } - - const providedKey = authHeader.slice(7); // Remove "Bearer " prefix - if (!providedKey) { - return { isValid: false }; - } - - // Timing-safe comparison using HMAC (Edge-compatible via Web Crypto API) - const encoder = new TextEncoder(); - const key = await crypto.subtle.importKey( - "raw", - encoder.encode("api-key-compare"), - { name: "HMAC", hash: "SHA-256" }, - false, - ["sign"] - ); - - const [providedMac, expectedMac] = await Promise.all([ - crypto.subtle.sign("HMAC", key, encoder.encode(providedKey)), - crypto.subtle.sign("HMAC", key, encoder.encode(NS_API_KEY)), - ]); - - const providedArr = new Uint8Array(providedMac); - const expectedArr = new Uint8Array(expectedMac); - - if (providedArr.length !== expectedArr.length) { - return { isValid: false }; - } - - let match = true; - for (let i = 0; i < providedArr.length; i++) { - if (providedArr[i] !== expectedArr[i]) { - match = false; - } - } - - return { isValid: match }; - } catch { + token?: string; +} { + const token = request.cookies.get(SESSION_COOKIE_NAME)?.value; + if (!token || token.trim().length === 0) { return { isValid: false }; } + return { + isValid: true, + token: token.trim(), + }; } function isPublicRoute(pathname: string): boolean { - // Remove base path prefix if present let normalizedPath = pathname; if (basePath && pathname.startsWith(basePath)) { normalizedPath = pathname.slice(basePath.length) || "/"; } - - // Check if it's a public route return publicRoutes.some((route) => normalizedPath.startsWith(route)); } @@ -209,63 +71,69 @@ export async function middleware(request: NextRequest) { let pathname = request.nextUrl.pathname; // STEP 0: Handle root path when base path is configured - // Redirect "/" to basePath (e.g., "/dashboard") if (basePath && pathname === "/" && basePath !== "") { return NextResponse.redirect(new URL(basePath, request.url)); } // STEP 1: Strip base path if present and rewrite the request - // This allows Next.js to serve pages at /dashboard/path while routing internally to /path if (basePath && pathname.startsWith(basePath)) { const strippedPath = pathname.slice(basePath.length) || '/'; - - // Create URL with stripped path for internal processing const url = request.nextUrl.clone(); url.pathname = strippedPath; - - // Update pathname for subsequent checks pathname = strippedPath; - - // Skip static files - rewrite and return immediately + if (isStaticFile(pathname)) { return NextResponse.rewrite(url); } - - // For non-static files, we'll rewrite at the end after auth checks - // Continue with auth checks using the stripped pathname } else { - // No base path or doesn't start with base path - // Skip static files if (isStaticFile(pathname)) { return NextResponse.next(); } } // STEP 2: Validate session from request cookies - const session = await validateSessionFromRequest(request); + const session = validateSessionFromRequest(request); // STEP 3: Handle root path after stripping base path - // When user visits /dashboard (with basePath=/dashboard), it becomes / internally - // Redirect authenticated users to the dashboard route, unauthenticated to login if (pathname === "/") { if (session.isValid) { - // Authenticated: redirect to dashboard route return NextResponse.redirect(new URL(`${basePath}/dashboard`, request.url)); } else { - // Not authenticated: redirect to login return NextResponse.redirect(new URL(`${basePath}/login`, request.url)); } } // STEP 4: Check if authenticated user is trying to access login page if (pathname === "/login" && session.isValid) { - // Redirect to dashboard return NextResponse.redirect(new URL(`${basePath}/dashboard`, request.url)); } - // STEP 6: Allow public routes (for non-authenticated users) + // STEP 5: Proxy API requests directly to backend when accessed on standalone dashboard port + if (pathname.startsWith("/api/")) { + // Public API routes + if (isPublicRoute(pathname)) { + return proxyToBackend(request, pathname); + } + + // Authenticated session + if (session.isValid) { + return proxyToBackend(request, pathname, session.token); + } + + // Bearer token fallback for external/MCP clients + const authHeader = request.headers.get("authorization"); + if (authHeader && authHeader.startsWith("Bearer ")) { + return proxyToBackend(request, pathname); + } + + return NextResponse.json( + { error: "Unauthorized: valid session or API key required" }, + { status: 401 } + ); + } + + // STEP 6: Allow public UI pages if (isPublicRoute(pathname)) { - // If we stripped base path, rewrite the request if (basePath && request.nextUrl.pathname.startsWith(basePath)) { const url = request.nextUrl.clone(); url.pathname = pathname; @@ -274,42 +142,13 @@ export async function middleware(request: NextRequest) { return NextResponse.next(); } - // STEP 7: Protected routes - require authentication + // STEP 7: Protected UI pages - redirect unauthenticated users to login if (!session.isValid) { - // For API routes, fall back to API key (Bearer token) authentication - // This allows programmatic access (e.g. from MCP server) without a session cookie - if (pathname.startsWith("/api/")) { - const apiKeyAuth = await validateApiKeyFromRequest(request); - if (apiKeyAuth.isValid) { - // Valid API key — allow the request through - if (isLocalApiRoute(pathname)) { - if (basePath && request.nextUrl.pathname.startsWith(basePath)) { - const url = request.nextUrl.clone(); - url.pathname = pathname; - return NextResponse.rewrite(url); - } - return NextResponse.next(); - } - return proxyToBackend(request, pathname); - } - // Invalid or missing API key on an API route — return 401 JSON instead of redirect - return NextResponse.json( - { error: "Unauthorized: valid session or API key required" }, - { status: 401 } - ); - } - - // Non-API routes: redirect to login const loginUrl = new URL(`${basePath}/login`, request.url); loginUrl.searchParams.set("callbackUrl", request.nextUrl.pathname); return NextResponse.redirect(loginUrl); } - // STEP 8: Proxy non-local API requests or perform normal next.js routing/basePath stripping - if (pathname.startsWith("/api/") && !isLocalApiRoute(pathname)) { - return proxyToBackend(request, pathname); - } - if (basePath && request.nextUrl.pathname.startsWith(basePath)) { const url = request.nextUrl.clone(); url.pathname = pathname; @@ -321,13 +160,6 @@ export async function middleware(request: NextRequest) { export const config = { matcher: [ - /* - * Match all request paths except for the ones starting with: - * - _next/static (static files) - * - _next/image (image optimization files) - * - favicon.ico (favicon file) - * - public files (images, etc.) - */ "/((?!_next/static|_next/image|favicon.ico|.*\\.png$|.*\\.jpg$|.*\\.svg$).*)", ], }; diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 47aece9..19a4ad0 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -1,12 +1,12 @@ { "name": "dashboard", - "version": "1.3.0", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dashboard", - "version": "1.3.0", + "version": "1.4.0", "dependencies": { "@monaco-editor/react": "^4.7.0", "@radix-ui/react-alert-dialog": "^1.1.15", diff --git a/dashboard/package.json b/dashboard/package.json index fec18b4..1f21730 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "dashboard", - "version": "1.3.0", + "version": "1.4.0", "private": true, "engines": { "node": ">=22.0.0" diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml deleted file mode 100644 index 019974f..0000000 --- a/docker-compose.dev.yaml +++ /dev/null @@ -1,371 +0,0 @@ -services: - # ============================================ - # Infrastructure Services - # ============================================ - mongo: - image: mongo:7.0 - container_name: mongo - command: [ "--replSet", "rs0", "--bind_ip_all", "--port", "27017" ] - ports: - - 27017:27017 - extra_hosts: - - "host.docker.internal:host-gateway" - healthcheck: - test: echo "try { rs.status() } catch (err) { rs.initiate({_id:'rs0',members:[{_id:0,host:'host.docker.internal:27017'}]}) }" | mongosh --port 27017 --quiet - interval: 5s - timeout: 30s - start_period: 0s - start_interval: 1s - retries: 30 - volumes: - - "mongo_data:/data/db" - - "mongo_config:/data/configdb" - networks: - - ns-network - - kafka: - image: apache/kafka-native - container_name: kafka - ports: - - "9092:9092" - environment: - # Configure listeners for both docker and host communication - KAFKA_LISTENERS: CONTROLLER://localhost:9091,HOST://0.0.0.0:9092,DOCKER://0.0.0.0:9093 - KAFKA_ADVERTISED_LISTENERS: HOST://localhost:9092,DOCKER://kafka:9093 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,DOCKER:PLAINTEXT,HOST:PLAINTEXT - - # Settings required for KRaft mode - KAFKA_NODE_ID: 1 - KAFKA_PROCESS_ROLES: broker,controller - KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9091 - - # Listener to use for broker-to-broker communication - KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER - - # Required for a single node cluster - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - - # Disable auto-topic creation - API server will create topics with correct partitions - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" - volumes: - - "kafka_data:/var/lib/kafka/data" - networks: - - ns-network - - kafka-ui: - image: kafbat/kafka-ui:main - container_name: kafka-ui - ports: - - 8080:8080 - environment: - DYNAMIC_CONFIG_ENABLED: "true" - KAFKA_CLUSTERS_0_NAME: local - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9093 - depends_on: - - kafka - networks: - - ns-network - - redis: - image: redis:7-alpine - container_name: redis - ports: - - "6379:6379" - command: redis-server --appendonly yes - volumes: - - "redis_data:/data" - networks: - - ns-network - healthcheck: - test: [ "CMD", "redis-cli", "ping" ] - interval: 5s - timeout: 3s - retries: 5 - - # ============================================ - # Observability Services - # ============================================ - - loki: - image: grafana/loki:2.9.0 - container_name: loki - ports: - - "3100:3100" - command: -config.file=/etc/loki/local-config.yaml - volumes: - - "loki_data:/loki" - networks: - - ns-network - healthcheck: - test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1" ] - interval: 10s - timeout: 5s - retries: 5 - - grafana: - image: grafana/grafana:10.2.0 - container_name: grafana - ports: - - "3001:3000" - environment: - - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - - GF_SECURITY_ADMIN_PASSWORD=admin - volumes: - - "grafana_data:/var/lib/grafana" - depends_on: - loki: - condition: service_healthy - networks: - - ns-network - - # ============================================ - # Application Services - # ============================================ - - # API Server - Handles HTTP requests for /notification and /notification/batch - api: - build: - context: . - dockerfile: Dockerfile - container_name: api - image: simplens-core:latest - ports: - - "3000:3000" - env_file: - - .env - environment: - NODE_ENV: ${NODE_ENV:-production} - PORT: ${PORT:-3000} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - NS_API_KEY: ${NS_API_KEY:-dev-api-key-change-in-production} - MAX_BATCH_REQ_LIMIT: ${MAX_BATCH_REQ_LIMIT:-1000} - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - DELAYED_PARTITION: ${DELAYED_PARTITION:-1} - NOTIFICATION_STATUS_PARTITION: ${NOTIFICATION_STATUS_PARTITION:-1} - SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} - volumes: - # Mount plugins directory for user-installed plugins (optional) - - plugin-data:/app/.plugins - - logs-data:/app/logs - - ./simplens.config.yaml:/app/simplens.config.yaml:ro - command: [ "node", "dist/api/server.js" ] - depends_on: - mongo: - condition: service_healthy - kafka: - condition: service_started - redis: - condition: service_healthy - loki: - condition: service_healthy - networks: - - ns-network - restart: unless-stopped - healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] - interval: 30s - timeout: 10s - retries: 3 - start_period: 10s - - # Background Worker - Polls outbox, publishes to Kafka, updates status in MongoDB - worker: - image: simplens-core:latest - extra_hosts: - - "host.docker.internal:host-gateway" - env_file: - - .env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - OUTBOX_POLL_INTERVAL_MS: ${OUTBOX_POLL_INTERVAL_MS:-5000} - OUTBOX_BATCH_SIZE: ${OUTBOX_BATCH_SIZE:-100} - OUTBOX_CLAIM_TIMEOUT_MS: ${OUTBOX_CLAIM_TIMEOUT_MS:-30000} - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - volumes: - - logs-data:/app/logs - command: [ "node", "dist/workers/worker.js" ] - depends_on: - mongo: - condition: service_healthy - kafka: - condition: service_started - redis: - condition: service_healthy - loki: - condition: service_healthy - api: - condition: service_healthy - networks: - - ns-network - restart: unless-stopped - - # Unified Notification Processor - Uses plugin system for all channels - notification-processor: - image: simplens-core:latest - env_file: - - .env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - # Plugin Configuration - path to simplens.config.yaml - SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml} - # Processor Mode - 'all' for all channels, or specific channel name - PROCESSOR_CHANNEL: ${PROCESSOR_CHANNEL:-all} - # Retry Configuration - MAX_RETRY_COUNT: ${MAX_RETRY_COUNT:-5} - IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} - PROCESSING_TTL_SECONDS: ${PROCESSING_TTL_SECONDS:-120} - # Logging - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - command: [ "node", "dist/processors/unified/unified.processor.js" ] - volumes: - # Mount plugins directory for user-installed plugins (optional) - - plugin-data:/app/.plugins - - logs-data:/app/logs - - ./simplens.config.yaml:/app/simplens.config.yaml:ro - depends_on: - mongo: - condition: service_healthy - kafka: - condition: service_started - redis: - condition: service_healthy - loki: - condition: service_healthy - api: - condition: service_healthy - networks: - - ns-network - restart: unless-stopped - - # Delayed Processor - Handles scheduled notifications via Redis ZSET queue - delayed-processor: - image: simplens-core:latest - env_file: - - .env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - BROKERS: kafka:9093 - # Delayed Queue Configuration - DELAYED_POLL_INTERVAL_MS: ${DELAYED_POLL_INTERVAL_MS:-1000} - DELAYED_BATCH_SIZE: ${DELAYED_BATCH_SIZE:-10} - MAX_POLLER_RETRIES: ${MAX_POLLER_RETRIES:-3} - # Logging - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - volumes: - - logs-data:/app/logs - command: [ "node", "dist/processors/delayed/delayed.processor.js" ] - depends_on: - mongo: - condition: service_healthy - kafka: - condition: service_started - redis: - condition: service_healthy - loki: - condition: service_healthy - api: - condition: service_healthy - networks: - - ns-network - restart: unless-stopped - - # Recovery Service - Detects stuck notifications and creates alerts - recovery: - image: simplens-core:latest - env_file: - - .env - environment: - NODE_ENV: ${NODE_ENV:-production} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - REDIS_URL: redis://redis:6379 - # Recovery Configuration - RECOVERY_POLL_INTERVAL_MS: ${RECOVERY_POLL_INTERVAL_MS:-60000} - PROCESSING_STUCK_THRESHOLD_MS: ${PROCESSING_STUCK_THRESHOLD_MS:-300000} - PENDING_STUCK_THRESHOLD_MS: ${PENDING_STUCK_THRESHOLD_MS:-300000} - RECOVERY_BATCH_SIZE: ${RECOVERY_BATCH_SIZE:-50} - MAX_RETRY_COUNT: ${MAX_RETRY_COUNT:-5} - # Cleanup Configuration - CLEANUP_RESOLVED_ALERTS_RETENTION_MS: ${CLEANUP_RESOLVED_ALERTS_RETENTION_MS:-86400000} - CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS: ${CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS:-86400000} - # Logging - LOKI_URL: http://loki:3100 - LOG_LEVEL: ${LOG_LEVEL:-info} - volumes: - - logs-data:/app/logs - command: [ "node", "dist/workers/recovery/recovery.service.js" ] - depends_on: - mongo: - condition: service_healthy - redis: - condition: service_healthy - loki: - condition: service_healthy - api: - condition: service_healthy - networks: - - ns-network - restart: unless-stopped - - # Admin Dashboard - Next.js web app for monitoring notifications - dashboard: - build: - context: ./dashboard - dockerfile: Dockerfile - container_name: dashboard - ports: - - "3002:3002" - env_file: - - .env - environment: - NODE_ENV: production - PORT: ${DASHBOARD_PORT:-3002} - MONGO_URI: mongodb://mongo:27017/notification_service?replicaSet=rs0 - AUTH_SECRET: ${AUTH_SECRET:-change-this-in-production} - ADMIN_USERNAME: ${ADMIN_USERNAME:-admin} - ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin} - AUTH_TRUST_HOST: ${AUTH_TRUST_HOST:-true} - # For sending notifications - API_BASE_URL: http://api:${PORT:-3000} - NS_API_KEY: ${NS_API_KEY:-dev-api-key-change-in-production} - WEBHOOK_HOST: dashboard - WEBHOOK_PORT: "3002" - depends_on: - mongo: - condition: service_healthy - api: - condition: service_started - networks: - - ns-network - restart: unless-stopped - -volumes: - mongo_data: - mongo_config: - kafka_data: - redis_data: - loki_data: - grafana_data: - plugin-data: - logs-data: - - -networks: - ns-network: - driver: bridge diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 702edbc..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,95 +0,0 @@ -services: - api: - image: ghcr.io/simplenotificationsystem/simplens-core:latest - container_name: api - ports: - - 3000:3000 - env_file: - - .env - volumes: - - plugin-data:/app/.plugins - - logs-data:/app/logs - - ./simplens.config.yaml:/app/simplens.config.yaml:ro - command: [ "node", "dist/api/server.js" ] - networks: - - simplens - restart: unless-stopped - healthcheck: - test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] - interval: 30s - timeout: 10s - retries: 3 - start_period: 10s - - worker: - image: ghcr.io/simplenotificationsystem/simplens-core:latest - env_file: - - .env - volumes: - - logs-data:/app/logs - command: [ "node", "dist/workers/worker.js" ] - networks: - - simplens - restart: unless-stopped - - notification_processor: - image: ghcr.io/simplenotificationsystem/simplens-core:latest - env_file: - - .env - volumes: - - plugin-data:/app/.plugins - - logs-data:/app/logs - - ./simplens.config.yaml:/app/simplens.config.yaml:ro - command: [ "node", "dist/processors/unified/unified.processor.js" ] - depends_on: - api: - condition: service_healthy - networks: - - simplens - restart: unless-stopped - - delayed_processor: - image: ghcr.io/simplenotificationsystem/simplens-core:latest - env_file: - - .env - volumes: - - logs-data:/app/logs - command: [ "node", "dist/processors/delayed/delayed.processor.js" ] - networks: - - simplens - restart: unless-stopped - - recovery: - image: ghcr.io/simplenotificationsystem/simplens-core:latest - env_file: - - .env - volumes: - - logs-data:/app/logs - command: [ "node", "dist/workers/recovery/recovery.service.js" ] - networks: - - simplens - restart: unless-stopped - - dashboard: - image: ghcr.io/simplenotificationsystem/simplens-dashboard:latest - ports: - - 3002:3002 - container_name: dashboard - env_file: - - .env - environment: - PORT: ${DASHBOARD_PORT:-3002} - API_BASE_URL: http://api:${PORT:-3000} - WEBHOOK_HOST: dashboard - WEBHOOK_PORT: ${DASHBOARD_PORT:-3002} - networks: - - simplens - restart: unless-stopped - -volumes: - plugin-data: - logs-data: - -networks: - simplens: - driver: bridge diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 0000000..696c3b8 --- /dev/null +++ b/docker-compose/README.md @@ -0,0 +1,123 @@ +# SimpleNS Multi-Environment Docker Compose Setup + +Run and test SimpleNS across three distinct environments (**Local**, **Development**, and **Master**) powered by a shared, persistent infrastructure stack. + +--- + +## 1. Quick Overview + +Instead of maintaining separate Kubernetes clusters or pods for routine branch testing, all environment switching is handled cleanly via Docker Compose: + +``` +docker-compose/ +├── docker-compose.infra.yaml # Shared base infrastructure (Mongo, Kafka, Redis, Loki, Grafana) +├── docker-compose.local.yaml # Application stack built from local source code +├── docker-compose.dev.yaml # Application stack using GHCR 'development' branch images +├── docker-compose.master.yaml # Application stack using GHCR 'latest' (master) release images +├── scripts/ +│ ├── switch-env.ps1 # Windows PowerShell switcher +│ └── switch-env.sh # Linux / macOS Bash switcher +└── README.md +``` + +### Key Highlights +- **Shared Infrastructure**: MongoDB replica set (`rs0`), Apache Kafka (KRaft), Redis, Loki, and Grafana run once and persist. +- **Zero Port Collisions**: All three environments use standard ports (`API: 3000`, `Dashboard: 3002`). The switching scripts cleanly terminate the previous environment before bringing up the new one. +- **Database Isolation**: + - `local` environment connects to `simplens_local` + - `dev` environment connects to `simplens_dev` + - `master` environment connects to `simplens_master` + +--- + +## 2. Usage & Switching Commands + +Use the unified switcher script from PowerShell (Windows) or Bash (Linux / macOS): + +### Windows (PowerShell) + +```powershell +# 1. Run Local source build (builds images from current working directory) +./docker-compose/scripts/switch-env.ps1 -Env local -Rebuild + +# 2. Run Development branch (pulls latest GHCR development image) +./docker-compose/scripts/switch-env.ps1 -Env dev -PullLatest + +# 3. Run Master branch (pulls latest GHCR master release) +./docker-compose/scripts/switch-env.ps1 -Env master -PullLatest + +# 4. Check active containers & service URLs +./docker-compose/scripts/switch-env.ps1 -Env status + +# 5. Stop application containers (keeps infrastructure running) +./docker-compose/scripts/switch-env.ps1 -Env down + +# 6. Stop everything (including infrastructure) +./docker-compose/scripts/switch-env.ps1 -Env infra-down +``` + +### Linux / macOS (Bash) + +```bash +# Make script executable +chmod +x ./docker-compose/scripts/switch-env.sh + +# 1. Run Local source build +./docker-compose/scripts/switch-env.sh local --rebuild + +# 2. Run Development branch +./docker-compose/scripts/switch-env.sh dev --pull + +# 3. Run Master branch +./docker-compose/scripts/switch-env.sh master --pull + +# 4. Check status +./docker-compose/scripts/switch-env.sh status + +# 5. Stop applications +./docker-compose/scripts/switch-env.sh down + +# 6. Stop everything +./docker-compose/scripts/switch-env.sh infra-down +``` + +--- + +## 3. Port & Service Directory + +When the stack is running, all services are accessible on your host machine: + +| Service | Host URL | Description / Credentials | +| :--- | :--- | :--- | +| **SimpleNS API** | [http://localhost:3000](http://localhost:3000) | Health endpoint: `/api/health` | +| **SimpleNS Dashboard** | [http://localhost:3002](http://localhost:3002) | Next.js admin management console | +| **Kafka UI** | [http://localhost:8081](http://localhost:8081) | Topic partitions, consumer groups, and lag | +| **Grafana** | [http://localhost:3001](http://localhost:3001) | User: `admin` / Password: `admin` | +| **MongoDB** | `mongodb://localhost:27017` | Direct database connection (`rs0`) | +| **Redis** | `redis://localhost:6379` | Direct Redis connection | + +--- + +## 4. Environment Comparison + +| Attribute | `local` | `dev` | `master` | +| :--- | :--- | :--- | :--- | +| **Image Source** | Local Dockerfile build | `ghcr.io/.../simplens-core:development` | `ghcr.io/.../simplens-core:latest` | +| **Database** | `simplens_local` | `simplens_dev` | `simplens_master` | +| **Use Case** | Active code development & testing | Testing pre-release features | Testing production-equivalent baseline | +| **Rebuild Flag** | `-Rebuild` / `--rebuild` | N/A | N/A | +| **Pull Flag** | N/A | `-PullLatest` / `--pull` | `-PullLatest` / `--pull` | + +--- + +## 5. Troubleshooting & FAQ + +### Port already in use (e.g. 3000 or 3002)? +Run `./docker-compose/scripts/switch-env.ps1 -Env down` to shut down any lingering application containers. If an old root Docker Compose is running, stop it via `docker compose down`. + +### Resetting all data? +To completely reset all databases and queues: +```powershell +./docker-compose/scripts/switch-env.ps1 -Env infra-down +docker volume prune -f +``` diff --git a/docker-compose/docker-compose.dev.yaml b/docker-compose/docker-compose.dev.yaml new file mode 100644 index 0000000..7701095 --- /dev/null +++ b/docker-compose/docker-compose.dev.yaml @@ -0,0 +1,102 @@ +# ============================================================================== +# SimpleNS Dev Environment +# Connects to shared 'simplens-infra' network and 'simplens_dev' database. +# ============================================================================== + +name: simplens-dev +services: + api: + image: ghcr.io/simplenotificationsystem/simplens-core:development + container_name: simplens-api-dev + ports: + - 3000:3000 + env_file: + - ../.env + volumes: + - logs-data-dev:/app/logs + command: [ "node", "dist/api/server.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "dev" + + worker: + image: ghcr.io/simplenotificationsystem/simplens-core:development + container_name: simplens-worker-dev + env_file: + - ../.env + volumes: + - logs-data-dev:/app/logs + command: [ "node", "dist/workers/worker.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "dev" + + + notification_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:development + container_name: simplens-notification-processor-dev + env_file: + - ../.env + volumes: + - logs-data-dev:/app/logs + command: [ "node", "dist/processors/unified/unified.processor.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "dev" + + delayed_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:development + container_name: simplens-delayed-processor-dev + env_file: + - ../.env + volumes: + - logs-data-dev:/app/logs + command: [ "node", "dist/processors/delayed/delayed.processor.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "dev" + + recovery: + image: ghcr.io/simplenotificationsystem/simplens-core:development + container_name: simplens-recovery-dev + env_file: + - ../.env + volumes: + - logs-data-dev:/app/logs + command: [ "node", "dist/workers/recovery/recovery.service.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "dev" + + dashboard: + image: ghcr.io/simplenotificationsystem/simplens-dashboard:development + ports: + - 3002:3002 + container_name: simplens-dashboard-dev + env_file: + - ../.env + environment: + PORT: ${DASHBOARD_PORT:-3002} + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "dev" + +volumes: + logs-data-dev: + +networks: + simplens-infra: + external: true + name: simplens-infra diff --git a/docker-compose.infra.yaml b/docker-compose/docker-compose.infra.yaml similarity index 58% rename from docker-compose.infra.yaml rename to docker-compose/docker-compose.infra.yaml index 1a3cb16..5b5483e 100644 --- a/docker-compose.infra.yaml +++ b/docker-compose/docker-compose.infra.yaml @@ -1,19 +1,21 @@ -# ============================================ -# INFRA_HOST: Set this in .env to your machine's IP/hostname when running -# infrastructure on a separate system from application services. -# Default: host.docker.internal (for same-system deployment) -# ============================================ +# ============================================================================== +# SimpleNS Shared Infrastructure Services +# Runs Mongo (Replica Set), Kafka (KRaft), Kafka-UI, Redis, Loki, Grafana +# Shared across all application environments (local, dev, master). +# ============================================================================== + +name: simplens-infra services: # ============================================ - # Infrastructure Services + # 1. MongoDB (Replica Set rs0 for Transactions) # ============================================ mongo: image: mongo:7.0 - container_name: mongo + container_name: simplens-infra-mongo command: [ "--replSet", "rs0", "--bind_ip_all", "--port", "27017" ] ports: - - 27017:27017 + - "27017:27017" extra_hosts: - "host.docker.internal:host-gateway" healthcheck: @@ -24,84 +26,98 @@ services: start_interval: 1s retries: 30 volumes: - - "mongo_data:/data/db" - - "mongo_config:/data/configdb" + - mongo_data:/data/db + - mongo_config:/data/configdb + networks: + - simplens-infra + restart: unless-stopped + # ============================================ + # 2. Apache Kafka (KRaft Native Mode) + # ============================================ kafka: image: apache/kafka-native - container_name: kafka + container_name: simplens-infra-kafka ports: - "9092:9092" environment: - # Configure listeners for both docker and host communication KAFKA_LISTENERS: CONTROLLER://localhost:9091,HOST://0.0.0.0:9092,DOCKER://0.0.0.0:9093 KAFKA_ADVERTISED_LISTENERS: HOST://${INFRA_HOST:-host.docker.internal}:9092,DOCKER://kafka:9093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,DOCKER:PLAINTEXT,HOST:PLAINTEXT - - # Settings required for KRaft mode KAFKA_NODE_ID: 1 KAFKA_PROCESS_ROLES: broker,controller KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9091 - - # Listener to use for broker-to-broker communication KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER - - # Required for a single node cluster KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - - # Disable auto-topic creation - API server will create topics with correct partitions KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" volumes: - - "kafka_data:/var/lib/kafka/data" + - kafka_data:/var/lib/kafka/data + networks: + - simplens-infra + restart: unless-stopped + # ============================================ + # 3. Kafka UI (Topic & Consumer Explorer) + # ============================================ kafka-ui: image: kafbat/kafka-ui:main - container_name: kafka-ui + container_name: simplens-infra-kafka-ui ports: - - 8080:8080 + - "8081:8080" environment: DYNAMIC_CONFIG_ENABLED: "true" KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9093 depends_on: - kafka + networks: + - simplens-infra + restart: unless-stopped + # ============================================ + # 4. Redis (Idempotency, Queue, Pub/Sub) + # ============================================ redis: image: redis:7-alpine - container_name: redis + container_name: simplens-infra-redis ports: - "6379:6379" command: redis-server --appendonly yes volumes: - - "redis_data:/data" + - redis_data:/data + networks: + - simplens-infra healthcheck: test: [ "CMD", "redis-cli", "ping" ] interval: 5s timeout: 3s retries: 5 + restart: unless-stopped # ============================================ - # Observability Services + # 5. Observability (Loki & Grafana) # ============================================ - loki: image: grafana/loki:2.9.0 - container_name: loki + container_name: simplens-infra-loki ports: - "3100:3100" command: -config.file=/etc/loki/local-config.yaml volumes: - - "loki_data:/loki" + - loki_data:/loki + networks: + - simplens-infra healthcheck: test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1" ] interval: 10s timeout: 5s retries: 5 + restart: unless-stopped grafana: image: grafana/grafana:10.2.0 - container_name: grafana + container_name: simplens-infra-grafana ports: - "3001:3000" environment: @@ -110,10 +126,13 @@ services: - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - GF_SECURITY_ADMIN_PASSWORD=admin volumes: - - "grafana_data:/var/lib/grafana" + - grafana_data:/var/lib/grafana depends_on: loki: condition: service_healthy + networks: + - simplens-infra + restart: unless-stopped volumes: mongo_data: @@ -121,4 +140,9 @@ volumes: kafka_data: redis_data: loki_data: - grafana_data: \ No newline at end of file + grafana_data: + +networks: + simplens-infra: + name: simplens-infra + driver: bridge diff --git a/docker-compose/docker-compose.local.yaml b/docker-compose/docker-compose.local.yaml new file mode 100644 index 0000000..0a11abe --- /dev/null +++ b/docker-compose/docker-compose.local.yaml @@ -0,0 +1,107 @@ +# ============================================================================== +# SimpleNS Local Environment +# Connects to shared 'simplens-infra' network and 'simplens_local' database. +# ============================================================================== + +name: simplens-local +services: + api: + build: + context: .. + dockerfile: Dockerfile + image: simplens-core:local + container_name: simplens-api-local + ports: + - 3000:3000 + env_file: + - ../.env + volumes: + - logs-data-local:/app/logs + command: [ "node", "dist/api/server.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "local" + + worker: + image: simplens-core:local + container_name: simplens-worker-local + env_file: + - ../.env + volumes: + - logs-data-local:/app/logs + command: [ "node", "dist/workers/worker.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "local" + + notification_processor: + image: simplens-core:local + container_name: simplens-notification-processor-local + env_file: + - ../.env + volumes: + - logs-data-local:/app/logs + command: [ "node", "dist/processors/unified/unified.processor.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "local" + + delayed_processor: + image: simplens-core:local + container_name: simplens-delayed-processor-local + env_file: + - ../.env + volumes: + - logs-data-local:/app/logs + command: [ "node", "dist/processors/delayed/delayed.processor.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "local" + + recovery: + image: simplens-core:local + container_name: simplens-recovery-local + env_file: + - ../.env + volumes: + - logs-data-local:/app/logs + command: [ "node", "dist/workers/recovery/recovery.service.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "local" + + dashboard: + build: + context: ../dashboard + dockerfile: Dockerfile + image: simplens-dashboard:local + ports: + - 3002:3002 + container_name: simplens-dashboard-local + env_file: + - ../.env + environment: + PORT: ${DASHBOARD_PORT:-3002} + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "local" + +volumes: + logs-data-local: + +networks: + simplens-infra: + external: true + name: simplens-infra diff --git a/docker-compose/docker-compose.master.yaml b/docker-compose/docker-compose.master.yaml new file mode 100644 index 0000000..4faca4f --- /dev/null +++ b/docker-compose/docker-compose.master.yaml @@ -0,0 +1,101 @@ +# ============================================================================== +# SimpleNS Master Environment +# Connects to shared 'simplens-infra' network and 'simplens_master' database. +# ============================================================================== + +name: simplens-master +services: + api: + image: ghcr.io/simplenotificationsystem/simplens-core:latest + container_name: simplens-api-master + ports: + - 3000:3000 + env_file: + - ../.env + volumes: + - logs-data-master:/app/logs + command: [ "node", "dist/api/server.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "master" + + worker: + image: ghcr.io/simplenotificationsystem/simplens-core:latest + container_name: simplens-worker-master + env_file: + - ../.env + volumes: + - logs-data-master:/app/logs + command: [ "node", "dist/workers/worker.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "master" + + notification_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:latest + container_name: simplens-notification-processor-master + env_file: + - ../.env + volumes: + - logs-data-master:/app/logs + command: [ "node", "dist/processors/unified/unified.processor.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "master" + + delayed_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:latest + container_name: simplens-delayed-processor-master + env_file: + - ../.env + volumes: + - logs-data-master:/app/logs + command: [ "node", "dist/processors/delayed/delayed.processor.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "master" + + recovery: + image: ghcr.io/simplenotificationsystem/simplens-core:latest + container_name: simplens-recovery-master + env_file: + - ../.env + volumes: + - logs-data-master:/app/logs + command: [ "node", "dist/workers/recovery/recovery.service.js" ] + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "master" + + dashboard: + image: ghcr.io/simplenotificationsystem/simplens-dashboard:latest + ports: + - 3002:3002 + container_name: simplens-dashboard-master + env_file: + - ../.env + environment: + PORT: ${DASHBOARD_PORT:-3002} + restart: unless-stopped + networks: + - simplens-infra + labels: + simplens.environment: "master" + +volumes: + logs-data-master: + +networks: + simplens-infra: + external: true + name: simplens-infra diff --git a/docker-compose/scripts/switch-env.ps1 b/docker-compose/scripts/switch-env.ps1 new file mode 100644 index 0000000..433c7d6 --- /dev/null +++ b/docker-compose/scripts/switch-env.ps1 @@ -0,0 +1,217 @@ +<# +.SYNOPSIS + Switches between SimpleNS Docker Compose environments (local, dev, master). + Ensures shared infrastructure is running while preventing container/port collisions. + +.PARAMETER Env + Environment to run: 'local', 'dev', 'master', 'status', 'down', 'infra-only', 'infra-down'. + Default: 'status' + +.PARAMETER Rebuild + (Local only) Rebuilds the local Docker images from source code before starting. + +.PARAMETER NoCache + (Local only) Rebuilds local Docker images without using Docker cache. + +.PARAMETER PullLatest + (Dev/Master only) Pulls the latest images from GHCR before starting. +#> + +param( + [ValidateSet("local", "dev", "master", "status", "down", "infra-only", "infra-down")] + [string]$Env = "status", + + [switch]$Rebuild, + [switch]$NoCache, + [switch]$PullLatest +) + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$ComposeDir = Split-Path -Parent $ScriptDir +$ProjectRoot = Split-Path -Parent $ComposeDir + +$InfraCompose = Join-Path $ComposeDir "docker-compose.infra.yaml" +$LocalCompose = Join-Path $ComposeDir "docker-compose.local.yaml" +$DevCompose = Join-Path $ComposeDir "docker-compose.dev.yaml" +$MasterCompose = Join-Path $ComposeDir "docker-compose.master.yaml" + +$EnvFile = Join-Path $ProjectRoot ".env" +if (-not (Test-Path $EnvFile)) { + $EnvFile = Join-Path $ProjectRoot ".env.example" +} + +function Ensure-Network { + $networkExists = docker network ls --filter "name=^simplens-infra$" --format "{{.Name}}" + if (-not $networkExists) { + Write-Host "[Network] Creating 'simplens-infra' bridge network..." -ForegroundColor Cyan + docker network create simplens-infra | Out-Null + } +} + +function Ensure-InfraImages { + Write-Host "[Infra] Verifying infrastructure Docker images..." -ForegroundColor Cyan + $images = @() + try { + $rawImages = docker compose -p infra --env-file $EnvFile -f $InfraCompose config --images 2>$null + if ($rawImages) { + $images = ($rawImages -split "`r?`n") | Where-Object { $_.Trim() -ne "" } | ForEach-Object { $_.Trim() } + } + } catch {} + + if (-not $images -or $images.Count -eq 0) { + $images = @( + "mongo:7.0", + "apache/kafka-native", + "kafbat/kafka-ui:main", + "redis:7-alpine", + "grafana/loki:2.9.0", + "grafana/grafana:10.2.0" + ) + } + + $missingImages = @() + foreach ($img in $images) { + $found = docker images -q $img 2>$null + if (-not $found) { + $missingImages += $img + } + } + + if ($missingImages.Count -gt 0) { + Write-Host "[Infra] Found $($missingImages.Count) missing infrastructure image(s). Pulling..." -ForegroundColor Yellow + foreach ($img in $missingImages) { + Write-Host "[Pull] Pulling '$img'..." -ForegroundColor Cyan + docker pull $img + } + Write-Host "[Infra] All required infrastructure images are downloaded." -ForegroundColor Green + } else { + Write-Host "[Infra] All infrastructure images are present locally." -ForegroundColor Green + } +} + +function Start-Infra { + Ensure-Network + Ensure-InfraImages + Write-Host "`n[Infra] Starting shared infrastructure services..." -ForegroundColor Cyan + docker compose -p infra --env-file $EnvFile -f $InfraCompose up -d + Write-Host "[Infra] Infrastructure services are up and healthy." -ForegroundColor Green +} + +function Stop-AppEnvironments { + param([string]$Except = "") + + if ($Except -ne "local") { + $localRunning = docker ps --filter "label=simplens.environment=local" --format "{{.ID}}" + if ($localRunning) { + Write-Host "[Stop] Stopping 'local' environment..." -ForegroundColor Yellow + docker compose -p simplens-local --env-file $EnvFile -f $LocalCompose down + } + } + if ($Except -ne "dev") { + $devRunning = docker ps --filter "label=simplens.environment=dev" --format "{{.ID}}" + if ($devRunning) { + Write-Host "[Stop] Stopping 'dev' environment..." -ForegroundColor Yellow + docker compose -p simplens-dev --env-file $EnvFile -f $DevCompose down + } + } + if ($Except -ne "master") { + $masterRunning = docker ps --filter "label=simplens.environment=master" --format "{{.ID}}" + if ($masterRunning) { + Write-Host "[Stop] Stopping 'master' environment..." -ForegroundColor Yellow + docker compose -p simplens-master --env-file $EnvFile -f $MasterCompose down + } + } +} + +function Show-Status { + Write-Host "`n============================================================" -ForegroundColor Cyan + Write-Host " SimpleNS Container Status " -ForegroundColor Cyan + Write-Host "============================================================" -ForegroundColor Cyan + + Write-Host "`n--- Infrastructure Containers (shared: infra) ---" -ForegroundColor Yellow + docker ps --filter "network=simplens-infra" --filter "name=simplens-infra" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" + + Write-Host "`n--- Active Application Containers ---" -ForegroundColor Yellow + docker ps --filter "network=simplens-infra" --filter "label=simplens.environment" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" + + Write-Host "`n--- Service URLs ---" -ForegroundColor Green + Write-Host " API Server: http://localhost:3000/api/health" -ForegroundColor White + Write-Host " Dashboard: http://localhost:3002" -ForegroundColor White + Write-Host " Kafka UI: http://localhost:8081" -ForegroundColor White + Write-Host " Grafana: http://localhost:3001 (User: admin / Pass: admin)" -ForegroundColor White + Write-Host "============================================================`n" -ForegroundColor Cyan +} + +switch ($Env) { + "local" { + Start-Infra + Stop-AppEnvironments -Except "local" + + if ($Rebuild -or $NoCache) { + Write-Host "`n[Build] Building local Docker images from source..." -ForegroundColor Cyan + $buildArgs = @("compose", "-p", "simplens-local", "--env-file", $EnvFile, "-f", $LocalCompose, "build") + if ($NoCache) { $buildArgs += "--no-cache" } + docker @buildArgs + } + + Write-Host "`n[Start] Starting 'local' environment..." -ForegroundColor Cyan + docker compose -p simplens-local --env-file $EnvFile -f $LocalCompose up -d + Write-Host "[OK] 'Local' environment is running!" -ForegroundColor Green + Show-Status + } + + "dev" { + Start-Infra + Stop-AppEnvironments -Except "dev" + + if ($PullLatest) { + Write-Host "`n[Pull] Pulling latest development images from GHCR..." -ForegroundColor Cyan + docker compose -p simplens-dev --env-file $EnvFile -f $DevCompose pull + } + + Write-Host "`n[Start] Starting 'dev' environment (GHCR development branch)..." -ForegroundColor Cyan + docker compose -p simplens-dev --env-file $EnvFile -f $DevCompose up -d + Write-Host "[OK] 'Dev' environment is running!" -ForegroundColor Green + Show-Status + } + + "master" { + Start-Infra + Stop-AppEnvironments -Except "master" + + if ($PullLatest) { + Write-Host "`n[Pull] Pulling latest master images from GHCR..." -ForegroundColor Cyan + docker compose -p simplens-master --env-file $EnvFile -f $MasterCompose pull + } + + Write-Host "`n[Start] Starting 'master' environment (GHCR latest release)..." -ForegroundColor Cyan + docker compose -p simplens-master --env-file $EnvFile -f $MasterCompose up -d + Write-Host "[OK] 'Master' environment is running!" -ForegroundColor Green + Show-Status + } + + "status" { + Show-Status + } + + "down" { + Write-Host "`n[Down] Stopping all active application environments..." -ForegroundColor Yellow + Stop-AppEnvironments + Write-Host "[OK] All application containers stopped. Infrastructure remains active." -ForegroundColor Green + } + + "infra-only" { + Stop-AppEnvironments + Start-Infra + Write-Host "[OK] Infrastructure services are running." -ForegroundColor Green + } + + "infra-down" { + Write-Host "`n[Infra-Down] Stopping all applications and shared infrastructure..." -ForegroundColor Yellow + Stop-AppEnvironments + docker compose -p infra --env-file $EnvFile -f $InfraCompose down + Write-Host "[OK] All SimpleNS containers stopped." -ForegroundColor Green + } +} diff --git a/docker-compose/scripts/switch-env.sh b/docker-compose/scripts/switch-env.sh new file mode 100644 index 0000000..51845d8 --- /dev/null +++ b/docker-compose/scripts/switch-env.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# ============================================================================== +# SimpleNS Docker Compose Environment Switcher (Linux / macOS / WSL) +# Switches between 'local', 'dev', and 'master' environments. +# ============================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +PROJECT_ROOT="$(cd "$COMPOSE_DIR/.." && pwd)" + +INFRA_COMPOSE="$COMPOSE_DIR/docker-compose.infra.yaml" +LOCAL_COMPOSE="$COMPOSE_DIR/docker-compose.local.yaml" +DEV_COMPOSE="$COMPOSE_DIR/docker-compose.dev.yaml" +MASTER_COMPOSE="$COMPOSE_DIR/docker-compose.master.yaml" + +ENV_FILE="$PROJECT_ROOT/.env" +if [ ! -f "$ENV_FILE" ]; then + ENV_FILE="$PROJECT_ROOT/.env.example" +fi + +ENV="${1:-status}" +REBUILD=false +NOCACHE=false +PULL_LATEST=false + +# Parse flags +shift || true +while [ $# -gt 0 ]; do + case "$1" in + --rebuild|-r) REBUILD=true ;; + --no-cache) NOCACHE=true; REBUILD=true ;; + --pull|-p) PULL_LATEST=true ;; + *) echo "Unknown flag: $1" >&2; exit 1 ;; + esac + shift +done + +ensure_network() { + if ! docker network inspect simplens-infra >/dev/null 2>&1; then + echo "[Network] Creating 'simplens-infra' bridge network..." + docker network create simplens-infra >/dev/null 2>&1 || true + fi +} + +ensure_infra_images() { + echo "[Infra] Verifying infrastructure Docker images..." + local images=( + "mongo:7.0" + "apache/kafka-native" + "kafbat/kafka-ui:main" + "redis:7-alpine" + "grafana/loki:2.9.0" + "grafana/grafana:10.2.0" + ) + + local missing_images=() + for img in "${images[@]}"; do + if ! docker image inspect "$img" >/dev/null 2>&1; then + missing_images+=("$img") + fi + done + + if [ ${#missing_images[@]} -gt 0 ]; then + echo "[Infra] Found ${#missing_images[@]} missing infrastructure image(s). Pulling..." + for img in "${missing_images[@]}"; do + echo "[Pull] Pulling '$img'..." + docker pull "$img" || true + done + echo "[Infra] All required infrastructure images are downloaded." + else + echo "[Infra] All infrastructure images are present locally." + fi +} + +start_infra() { + ensure_network + ensure_infra_images + echo "[Infra] Starting shared infrastructure services..." + docker compose -p infra --env-file "$ENV_FILE" -f "$INFRA_COMPOSE" up -d + echo "[Infra] Infrastructure services are up and healthy." +} + +stop_app_environments() { + local except="${1:-}" + + if [ "$except" != "local" ]; then + local local_running + local_running=$(docker ps -q --filter "label=simplens.environment=local" 2>/dev/null || true) + if [ -n "$local_running" ]; then + echo "[Stop] Stopping 'local' environment..." + docker compose -p simplens-local --env-file "$ENV_FILE" -f "$LOCAL_COMPOSE" down 2>/dev/null || true + fi + fi + + if [ "$except" != "dev" ]; then + local dev_running + dev_running=$(docker ps -q --filter "label=simplens.environment=dev" 2>/dev/null || true) + if [ -n "$dev_running" ]; then + echo "[Stop] Stopping 'dev' environment..." + docker compose -p simplens-dev --env-file "$ENV_FILE" -f "$DEV_COMPOSE" down 2>/dev/null || true + fi + fi + + if [ "$except" != "master" ]; then + local master_running + master_running=$(docker ps -q --filter "label=simplens.environment=master" 2>/dev/null || true) + if [ -n "$master_running" ]; then + echo "[Stop] Stopping 'master' environment..." + docker compose -p simplens-master --env-file "$ENV_FILE" -f "$MASTER_COMPOSE" down 2>/dev/null || true + fi + fi +} + +show_status() { + echo "" + echo "============================================================" + echo " SimpleNS Container Status " + echo "============================================================" + echo "" + echo "--- Infrastructure Containers (shared: infra) ---" + docker ps --filter "network=simplens-infra" --filter "name=simplens-infra" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" + echo "" + echo "--- Active Application Containers ---" + docker ps --filter "network=simplens-infra" --filter "label=simplens.environment" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" + echo "" + echo "--- Service URLs ---" + echo " API Server: http://localhost:3000/api/health" + echo " Dashboard: http://localhost:3002" + echo " Kafka UI: http://localhost:8081" + echo " Grafana: http://localhost:3001 (User: admin / Pass: admin)" + echo "============================================================" + echo "" +} + +case "$ENV" in + local) + start_infra + stop_app_environments "local" + if [ "$REBUILD" = true ]; then + echo "[Build] Building local Docker images from source..." + if [ "$NOCACHE" = true ]; then + docker compose -p simplens-local --env-file "$ENV_FILE" -f "$LOCAL_COMPOSE" build --no-cache + else + docker compose -p simplens-local --env-file "$ENV_FILE" -f "$LOCAL_COMPOSE" build + fi + fi + echo "[Start] Starting 'local' environment..." + docker compose -p simplens-local --env-file "$ENV_FILE" -f "$LOCAL_COMPOSE" up -d + echo "[OK] 'Local' environment is running!" + show_status + ;; + + dev) + start_infra + stop_app_environments "dev" + if [ "$PULL_LATEST" = true ]; then + echo "[Pull] Pulling latest development images from GHCR..." + docker compose -p simplens-dev --env-file "$ENV_FILE" -f "$DEV_COMPOSE" pull + fi + echo "[Start] Starting 'dev' environment..." + docker compose -p simplens-dev --env-file "$ENV_FILE" -f "$DEV_COMPOSE" up -d + echo "[OK] 'Dev' environment is running!" + show_status + ;; + + master) + start_infra + stop_app_environments "master" + if [ "$PULL_LATEST" = true ]; then + echo "[Pull] Pulling latest master images from GHCR..." + docker compose -p simplens-master --env-file "$ENV_FILE" -f "$MASTER_COMPOSE" pull + fi + echo "[Start] Starting 'master' environment..." + docker compose -p simplens-master --env-file "$ENV_FILE" -f "$MASTER_COMPOSE" up -d + echo "[OK] 'Master' environment is running!" + show_status + ;; + + status) + show_status + ;; + + down) + echo "[Down] Stopping all active application environments..." + stop_app_environments "" + echo "[OK] All application containers stopped. Infrastructure remains active." + ;; + + infra-only) + stop_app_environments "" + start_infra + echo "[OK] Infrastructure services are running." + ;; + + infra-down) + echo "[Infra-Down] Stopping all applications and shared infrastructure..." + stop_app_environments "" + docker compose -p infra --env-file "$ENV_FILE" -f "$INFRA_COMPOSE" down + echo "[OK] All SimpleNS containers stopped." + ;; + + *) + echo "Usage: $0 {local|dev|master|status|down|infra-only|infra-down} [--rebuild] [--no-cache] [--pull]" + exit 1 + ;; +esac diff --git a/k8s/README.md b/k8s/README.md deleted file mode 100644 index 1c61a90..0000000 --- a/k8s/README.md +++ /dev/null @@ -1,197 +0,0 @@ -# SimpleNS Kubernetes Setup Guide - -Run SimpleNS in a local Kubernetes cluster (Docker Desktop or Kind) with testing environments for **Local builds**, **Master branch (Prod)**, and **Development branch**, all sharing a single infrastructure pod. - ---- - -## 1. Quick Start in 4 Steps - -### Step 0: Create the Kind Cluster (One-time Setup) -Ensure you have `kind` installed (`winget install Kubernetes.kind` or `choco install kind`), then create the cluster: -```powershell -kind create cluster --name simplens --config k8s/kind-config.yaml -``` - ---- - -### Step 1: Build Local Images -Build the Docker images for your local code: -```powershell -./k8s/scripts/win/build-local.ps1 -``` -> **Note:** The script automatically detects the running Kind cluster (from your kubectl context or Kind cluster list), loads `simplens-core:local` and `simplens-dashboard:local` into Kind, and prunes dangling `:` images to prevent duplicate image clutter in Docker Desktop. -> If you need a clean rebuild without Docker cache, pass `-NoCache`: -> ```powershell -> ./k8s/scripts/win/build-local.ps1 -NoCache -> ``` - ---- - -### Step 2: Deploy to Kubernetes -Deploy the shared infrastructure pod and start the local environment: -```powershell -./k8s/scripts/win/deploy-all.ps1 -ActiveEnv local -``` -This script automatically: -- Creates the `simplens` namespace -- Injects your `.env` file as a Kubernetes Secret (`app-env`) -- Provisions persistent storage for Mongo, Kafka, Redis, and logs -- Starts the **Infra Pod** (Mongo, Kafka, Kafka-UI, Redis, Loki, Grafana) -- Starts the **Local App Pod** (API, Worker, Processor, Delayed Processor, Recovery, Dashboard) - ---- - -### Step 3: Open Port Forwarding (Access in Browser) -In a separate terminal tab, start the port forwarder: -```powershell -./k8s/scripts/win/port-forward.ps1 -``` -*(The script automatically cleans up previous port-forward processes and avoids port conflicts with Kind NodePorts).* - -Now open your browser: - -| Service | Browser URL | Credentials / Notes | -| :--- | :--- | :--- | -| **SimpleNS Dashboard** | [http://localhost:3002](http://localhost:3002) *(or NodePort :30302)* | Admin web interface | -| **API Health Check** | [http://localhost:3000/api/health](http://localhost:3000/api/health) *(or NodePort :30300)* | Returns `{"status":"ok"}` | -| **Kafka UI** | [http://localhost:8080](http://localhost:8080) *(or NodePort :30080)* | Topic & message explorer | -| **Grafana** | [http://localhost:3001](http://localhost:3001) *(or NodePort :30001)* | User: `admin` / Password: `admin` | - -*(Keep this terminal open while testing. Press `Ctrl + C` to stop port forwarding).* - ---- - -## 2. Iterating on Local Code (Hot Rebuilding) - -Whenever you make code changes and want to update the running local pod: -```powershell -./k8s/scripts/win/switch-env.ps1 -Env local -PullLatest -``` -This single command will: -1. Rebuild the local Docker images from your current source code. -2. Load the updated images into your Kind cluster. -3. Clean up dangling images so duplicates don't accumulate. -4. Trigger a rolling restart of `app-local` and wait for the new pod rollout to complete before returning. - -> **Tip:** If updating Dashboard UI components, do a hard refresh in the browser (`Ctrl + Shift + R` or `Ctrl + F5`) to bypass cached Next.js static bundles. - ---- - -## 3. Helper Scripts Reference - -All scripts live in `k8s/scripts/win/` and are designed to be simple: - -### `build-local.ps1` -Builds `simplens-core:local` and `simplens-dashboard:local` from your current source code, loads them into Kind, and prunes dangling images. -```powershell -# Standard build: -./k8s/scripts/win/build-local.ps1 - -# Clean build without Docker cache: -./k8s/scripts/win/build-local.ps1 -NoCache - -# Explicit cluster target if multiple exist: -./k8s/scripts/win/build-local.ps1 -ClusterName simplens -``` - ---- - -### `deploy-all.ps1` -Applies all manifests, sets up storage, reads `.env`, starts the infra pod, and brings up the chosen app environment. -```powershell -# Start with Local build active (default): -./k8s/scripts/win/deploy-all.ps1 -ActiveEnv local - -# Or start with Master (Prod GHCR) active: -./k8s/scripts/win/deploy-all.ps1 -ActiveEnv master - -# Or start with Development (GHCR) active: -./k8s/scripts/win/deploy-all.ps1 -ActiveEnv development -``` - ---- - -### `port-forward.ps1` -Tunnels ports from the Kubernetes cluster to your Windows machine (`localhost`). Automatically terminates stale `kubectl port-forward` processes and detects active NodePorts to prevent port collisions. -```powershell -# Forwards ports for ALL currently running environments + Kafka UI + Grafana: -./k8s/scripts/win/port-forward.ps1 - -# Or target only a specific environment: -./k8s/scripts/win/port-forward.ps1 -Env local -./k8s/scripts/win/port-forward.ps1 -Env master -./k8s/scripts/win/port-forward.ps1 -Env development -``` - ---- - -### `switch-env.ps1` -Switch which application environment is actively processing notifications. -*(Because all 3 environments connect to the same shared Kafka queue, running one environment at a time prevents queue collisions)*: - -```powershell -# Switch to Local build (uses existing build): -./k8s/scripts/win/switch-env.ps1 -Env local - -# Rebuild local code, load into Kind, restart pod, and wait for readiness: -./k8s/scripts/win/switch-env.ps1 -Env local -PullLatest - -# Rebuild without Docker cache: -./k8s/scripts/win/switch-env.ps1 -Env local -PullLatest -NoCache - -# Switch to Development (uses cached image): -./k8s/scripts/win/switch-env.ps1 -Env development - -# Pull latest image from GHCR on demand, then switch: -./k8s/scripts/win/switch-env.ps1 -Env development -PullLatest -./k8s/scripts/win/switch-env.ps1 -Env master -PullLatest - -# Check current pod status: -./k8s/scripts/win/switch-env.ps1 -Env status -``` - ---- - -## 4. Port Allocations Reference - -When running `./k8s/scripts/win/port-forward.ps1`, ports are mapped as follows: - -| Environment | Dashboard | API | Description | -| :--- | :--- | :--- | :--- | -| **Local** | `http://localhost:30302` *(or :3002)* | `http://localhost:30300` *(or :3000)* | Your locally built code | -| **Master** | `http://localhost:30102` | `http://localhost:30100` | Latest release from `master` branch | -| **Development** | `http://localhost:30202` | `http://localhost:30200` | Latest build from `development` branch | -| **Kafka UI** | `http://localhost:8080` | — | Inspect Kafka topics & consumers | -| **Grafana** | [http://localhost:3001](http://localhost:3001) | — | Observability & Loki logs | - ---- - -## 5. How Environment Variables Work (`.env`) - -You do not need to manually copy environment variables into Kubernetes manifests: -1. `deploy-all.ps1` reads your root `.env` file (or `.env.example` as fallback). -2. It creates a Kubernetes Secret called `app-env` inside the `simplens` namespace. -3. Every container in the App Pod automatically imports these variables via `envFrom`. -4. Cluster-internal URLs (like `mongo:27017`, `kafka:9093`, `redis:6379`) and environment-specific database names (`simplens_local`, `simplens_master`, `simplens_development`) are automatically applied as overrides so each environment stays isolated. - ---- - -## 6. Useful Debugging Commands - -```powershell -# See all pods and their status: -kubectl get pods -n simplens - -# View logs from a specific container: -kubectl logs -n simplens deploy/app-local -c api -f -kubectl logs -n simplens deploy/app-local -c worker -f -kubectl logs -n simplens deploy/app-local -c notification-processor -f -kubectl logs -n simplens deploy/simplens-infra -c mongo -f -kubectl logs -n simplens deploy/simplens-infra -c kafka -f - -# Restart a deployment: -kubectl rollout restart deployment/app-local -n simplens -kubectl rollout restart deployment/simplens-infra -n simplens -``` - ---- \ No newline at end of file diff --git a/k8s/apps/app-development.yaml b/k8s/apps/app-development.yaml deleted file mode 100644 index cdce43f..0000000 --- a/k8s/apps/app-development.yaml +++ /dev/null @@ -1,236 +0,0 @@ -# ============================================================================== -# SimpleNS Development Environment (GHCR Development Branch Images) -# Single Pod containing 6 containers: API, Worker, Unified Processor, -# Delayed Processor, Recovery Service, and Next.js Dashboard. -# ============================================================================== -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app-development - namespace: simplens - labels: - app.kubernetes.io/name: app-development - app.kubernetes.io/env: development -spec: - replicas: 0 # Start scaled down to prevent Kafka queue conflict until switched - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: app-development - template: - metadata: - labels: - app.kubernetes.io/name: app-development - app.kubernetes.io/env: development - spec: - containers: - # ============================================ - # 1. API Server - # ============================================ - - name: api - image: ghcr.io/simplenotificationsystem/simplens-core:development - imagePullPolicy: IfNotPresent - command: ["node", "dist/api/server.js"] - ports: - - containerPort: 3000 - name: http-api - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: PORT - value: "3000" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/1" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - readinessProbe: - httpGet: - path: /api/health - port: 3000 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - - # ============================================ - # 2. Outbox Worker - # ============================================ - - name: worker - image: ghcr.io/simplenotificationsystem/simplens-core:development - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/worker.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/1" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 3. Unified Notification Processor - # ============================================ - - name: notification-processor - image: ghcr.io/simplenotificationsystem/simplens-core:development - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/unified/unified.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/1" - - name: BROKERS - value: "kafka:9093" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - - name: PROCESSOR_CHANNEL - value: "all" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 4. Delayed Queue Processor - # ============================================ - - name: delayed-processor - image: ghcr.io/simplenotificationsystem/simplens-core:development - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/delayed/delayed.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/1" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 5. Recovery Service (Stuck notification recovery & cron) - # ============================================ - - name: recovery - image: ghcr.io/simplenotificationsystem/simplens-core:development - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/recovery/recovery.service.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/1" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 6. Admin Dashboard (Next.js) - # ============================================ - - name: dashboard - image: ghcr.io/simplenotificationsystem/simplens-dashboard:development - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3002 - name: http-dash - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: PORT - value: "3002" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_development?replicaSet=rs0" - - name: API_BASE_URL - value: "http://localhost:3000" - - name: WEBHOOK_HOST - value: "localhost" - - name: WEBHOOK_PORT - value: "3002" - - name: HTTPS_COOKIE - value: "false" - - volumes: - - name: simplens-config-vol - configMap: - name: simplens-config - - name: plugin-data - emptyDir: {} - - name: logs-data - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: app-development - namespace: simplens - labels: - app.kubernetes.io/name: app-development - app.kubernetes.io/env: development -spec: - type: NodePort - selector: - app.kubernetes.io/name: app-development - ports: - - name: http-api - port: 3000 - targetPort: 3000 - nodePort: 30200 - - name: http-dashboard - port: 3002 - targetPort: 3002 - nodePort: 30202 diff --git a/k8s/apps/app-local.yaml b/k8s/apps/app-local.yaml deleted file mode 100644 index 65b5ce1..0000000 --- a/k8s/apps/app-local.yaml +++ /dev/null @@ -1,236 +0,0 @@ -# ============================================================================== -# SimpleNS Local Environment (Locally Built Docker Images) -# Single Pod containing 6 containers: API, Worker, Unified Processor, -# Delayed Processor, Recovery Service, and Next.js Dashboard. -# ============================================================================== -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app-local - namespace: simplens - labels: - app.kubernetes.io/name: app-local - app.kubernetes.io/env: local -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: app-local - template: - metadata: - labels: - app.kubernetes.io/name: app-local - app.kubernetes.io/env: local - spec: - containers: - # ============================================ - # 1. API Server - # ============================================ - - name: api - image: simplens-core:local - imagePullPolicy: IfNotPresent - command: ["node", "dist/api/server.js"] - ports: - - containerPort: 3000 - name: http-api - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: PORT - value: "3000" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/2" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - readinessProbe: - httpGet: - path: /api/health - port: 3000 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - - # ============================================ - # 2. Outbox Worker - # ============================================ - - name: worker - image: simplens-core:local - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/worker.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/2" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 3. Unified Notification Processor - # ============================================ - - name: notification-processor - image: simplens-core:local - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/unified/unified.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/2" - - name: BROKERS - value: "kafka:9093" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - - name: PROCESSOR_CHANNEL - value: "all" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 4. Delayed Queue Processor - # ============================================ - - name: delayed-processor - image: simplens-core:local - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/delayed/delayed.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/2" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 5. Recovery Service (Stuck notification recovery & cron) - # ============================================ - - name: recovery - image: simplens-core:local - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/recovery/recovery.service.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/2" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 6. Admin Dashboard (Next.js) - # ============================================ - - name: dashboard - image: simplens-dashboard:local - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3002 - name: http-dash - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "development" - - name: PORT - value: "3002" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_local?replicaSet=rs0" - - name: API_BASE_URL - value: "http://localhost:3000" - - name: WEBHOOK_HOST - value: "localhost" - - name: WEBHOOK_PORT - value: "3002" - - name: HTTPS_COOKIE - value: "false" - - volumes: - - name: simplens-config-vol - configMap: - name: simplens-config - - name: plugin-data - emptyDir: {} - - name: logs-data - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: app-local - namespace: simplens - labels: - app.kubernetes.io/name: app-local - app.kubernetes.io/env: local -spec: - type: NodePort - selector: - app.kubernetes.io/name: app-local - ports: - - name: http-api - port: 3000 - targetPort: 3000 - nodePort: 30300 - - name: http-dashboard - port: 3002 - targetPort: 3002 - nodePort: 30302 diff --git a/k8s/apps/app-master.yaml b/k8s/apps/app-master.yaml deleted file mode 100644 index 3ab2d0b..0000000 --- a/k8s/apps/app-master.yaml +++ /dev/null @@ -1,236 +0,0 @@ -# ============================================================================== -# SimpleNS Master Environment (Production - GHCR Master Images) -# Single Pod containing 6 containers: API, Worker, Unified Processor, -# Delayed Processor, Recovery Service, and Next.js Dashboard. -# ============================================================================== -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app-master - namespace: simplens - labels: - app.kubernetes.io/name: app-master - app.kubernetes.io/env: master -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: app-master - template: - metadata: - labels: - app.kubernetes.io/name: app-master - app.kubernetes.io/env: master - spec: - containers: - # ============================================ - # 1. API Server - # ============================================ - - name: api - image: ghcr.io/simplenotificationsystem/simplens-core:latest - imagePullPolicy: IfNotPresent - command: ["node", "dist/api/server.js"] - ports: - - containerPort: 3000 - name: http-api - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: PORT - value: "3000" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/0" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - readinessProbe: - httpGet: - path: /api/health - port: 3000 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - - # ============================================ - # 2. Outbox Worker - # ============================================ - - name: worker - image: ghcr.io/simplenotificationsystem/simplens-core:latest - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/worker.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/0" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 3. Unified Notification Processor - # ============================================ - - name: notification-processor - image: ghcr.io/simplenotificationsystem/simplens-core:latest - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/unified/unified.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/0" - - name: BROKERS - value: "kafka:9093" - - name: SIMPLENS_CONFIG_PATH - value: "/app/simplens.config.yaml" - - name: PROCESSOR_CHANNEL - value: "all" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: simplens-config-vol - mountPath: /app/simplens.config.yaml - subPath: simplens.config.yaml - - name: plugin-data - mountPath: /app/.plugins - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 4. Delayed Queue Processor - # ============================================ - - name: delayed-processor - image: ghcr.io/simplenotificationsystem/simplens-core:latest - imagePullPolicy: IfNotPresent - command: ["node", "dist/processors/delayed/delayed.processor.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/0" - - name: BROKERS - value: "kafka:9093" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 5. Recovery Service (Stuck notification recovery & cron) - # ============================================ - - name: recovery - image: ghcr.io/simplenotificationsystem/simplens-core:latest - imagePullPolicy: IfNotPresent - command: ["node", "dist/workers/recovery/recovery.service.js"] - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: REDIS_URL - value: "redis://redis:6379/0" - - name: LOKI_URL - value: "http://loki:3100" - volumeMounts: - - name: logs-data - mountPath: /app/logs - - # ============================================ - # 6. Admin Dashboard (Next.js) - # ============================================ - - name: dashboard - image: ghcr.io/simplenotificationsystem/simplens-dashboard:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3002 - name: http-dash - envFrom: - - secretRef: - name: app-env - env: - - name: NODE_ENV - value: "production" - - name: PORT - value: "3002" - - name: MONGO_URI - value: "mongodb://mongo:27017/simplens_master?replicaSet=rs0" - - name: API_BASE_URL - value: "http://localhost:3000" - - name: WEBHOOK_HOST - value: "localhost" - - name: WEBHOOK_PORT - value: "3002" - - name: HTTPS_COOKIE - value: "false" - - volumes: - - name: simplens-config-vol - configMap: - name: simplens-config - - name: plugin-data - emptyDir: {} - - name: logs-data - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: app-master - namespace: simplens - labels: - app.kubernetes.io/name: app-master - app.kubernetes.io/env: master -spec: - type: NodePort - selector: - app.kubernetes.io/name: app-master - ports: - - name: http-api - port: 3000 - targetPort: 3000 - nodePort: 30100 - - name: http-dashboard - port: 3002 - targetPort: 3002 - nodePort: 30102 diff --git a/k8s/base/configmap-simplens.yaml b/k8s/base/configmap-simplens.yaml deleted file mode 100644 index 3e0c2d4..0000000 --- a/k8s/base/configmap-simplens.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: simplens-config - namespace: simplens -data: - simplens.config.yaml: | - # ============================================ - # SimpleNS Configuration - # ============================================ - providers: - # mock - @simplens/mock - - package: "@simplens/mock" - id: mock - credentials: - {} # No credentials required - options: - priority: 1 # Higher = preferred - rateLimit: - maxTokens: 100 # Token bucket capacity - refillRate: 10 # Tokens per interval - refillInterval: second # Options: second, minute, hour, day - - # mock-fb - @simplens/mock - - package: "@simplens/mock" - id: mock-fb - credentials: - {} # No credentials required - options: - priority: 1 # Higher = preferred - rateLimit: - maxTokens: 100 # Token bucket capacity - refillRate: 10 # Tokens per interval - refillInterval: second # Options: second, minute, hour, day - - # nodemailer-gmail - @simplens/nodemailer-gmail - - package: "@simplens/nodemailer-gmail" - id: nodemailer-gmail - credentials: - EMAIL_USER: ${EMAIL_USER} - EMAIL_PASS: ${EMAIL_PASS} - optionalConfig: - EMAIL_HOST: ${EMAIL_HOST} - EMAIL_PORT: ${EMAIL_PORT} - EMAIL_FROM: ${EMAIL_FROM} - options: - priority: 1 - rateLimit: - maxTokens: 100 - refillRate: 10 - refillInterval: second - - channels: - mock: - default: mock - fallback: mock-fb - email: - default: nodemailer-gmail diff --git a/k8s/base/namespace.yaml b/k8s/base/namespace.yaml deleted file mode 100644 index 1df6ef8..0000000 --- a/k8s/base/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: simplens - labels: - app.kubernetes.io/name: simplens diff --git a/k8s/infra/pod-infra.yaml b/k8s/infra/pod-infra.yaml deleted file mode 100644 index 6aae517..0000000 --- a/k8s/infra/pod-infra.yaml +++ /dev/null @@ -1,201 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplens-infra - namespace: simplens - labels: - app.kubernetes.io/name: simplens-infra - app.kubernetes.io/component: infrastructure -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: simplens-infra - template: - metadata: - labels: - app.kubernetes.io/name: simplens-infra - app.kubernetes.io/component: infrastructure - spec: - hostAliases: - - ip: "127.0.0.1" - hostnames: - - "mongo" - - "kafka" - - "redis" - - "loki" - containers: - # ============================================ - # 1. MongoDB (7.0 with replica set rs0) - # ============================================ - - name: mongo - image: mongo:7.0 - command: - - mongod - - --replSet - - rs0 - - --bind_ip_all - - --port - - "27017" - ports: - - containerPort: 27017 - name: mongo - volumeMounts: - - name: mongo-data - mountPath: /data/db - - name: mongo-config - mountPath: /data/configdb - readinessProbe: - exec: - command: - - mongosh - - --port - - "27017" - - --quiet - - --eval - - "try { if (rs.status().ok === 1) quit(0); } catch(e) {} try { rs.initiate({_id:'rs0',members:[{_id:0,host:'mongo:27017'}]}); quit(0); } catch(err) { quit(1); }" - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 10 - failureThreshold: 10 - - # ============================================ - # 2. Apache Kafka (native KRaft mode) - # ============================================ - - name: kafka - image: apache/kafka-native - ports: - - containerPort: 9091 - name: controller - - containerPort: 9092 - name: host - - containerPort: 9093 - name: internal - env: - - name: KAFKA_LISTENERS - value: "CONTROLLER://localhost:9091,INTERNAL://0.0.0.0:9093,HOST://0.0.0.0:9092" - - name: KAFKA_ADVERTISED_LISTENERS - value: "INTERNAL://kafka:9093,HOST://localhost:9092" - - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP - value: "CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,HOST:PLAINTEXT" - - name: KAFKA_NODE_ID - value: "1" - - name: KAFKA_PROCESS_ROLES - value: "broker,controller" - - name: KAFKA_CONTROLLER_LISTENER_NAMES - value: "CONTROLLER" - - name: KAFKA_CONTROLLER_QUORUM_VOTERS - value: "1@localhost:9091" - - name: KAFKA_INTER_BROKER_LISTENER_NAME - value: "INTERNAL" - - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR - value: "1" - - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE - value: "false" - volumeMounts: - - name: kafka-data - mountPath: /var/lib/kafka/data - - # ============================================ - # 3. Kafka UI (Management dashboard) - # ============================================ - - name: kafka-ui - image: kafbat/kafka-ui:main - ports: - - containerPort: 8080 - name: http-kafkaui - env: - - name: DYNAMIC_CONFIG_ENABLED - value: "true" - - name: KAFKA_CLUSTERS_0_NAME - value: "local" - - name: KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS - value: "localhost:9093" - - # ============================================ - # 4. Redis (7-alpine with appendonly) - # ============================================ - - name: redis - image: redis:7-alpine - command: - - redis-server - - --appendonly - - "yes" - ports: - - containerPort: 6379 - name: redis - volumeMounts: - - name: redis-data - mountPath: /data - readinessProbe: - exec: - command: - - redis-cli - - ping - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 3 - - # ============================================ - # 5. Loki (Log aggregation) - # ============================================ - - name: loki - image: grafana/loki:2.9.0 - args: - - -config.file=/etc/loki/local-config.yaml - ports: - - containerPort: 3100 - name: http-loki - volumeMounts: - - name: loki-data - mountPath: /loki - readinessProbe: - httpGet: - path: /ready - port: 3100 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - - # ============================================ - # 6. Grafana (Dashboard & observability) - # ============================================ - - name: grafana - image: grafana/grafana:10.2.0 - ports: - - containerPort: 3000 - name: http-grafana - env: - - name: GF_PATHS_PROVISIONING - value: "/etc/grafana/provisioning" - - name: GF_AUTH_ANONYMOUS_ENABLED - value: "true" - - name: GF_AUTH_ANONYMOUS_ORG_ROLE - value: "Admin" - - name: GF_SECURITY_ADMIN_PASSWORD - value: "admin" - volumeMounts: - - name: grafana-data - mountPath: /var/lib/grafana - - volumes: - - name: mongo-data - persistentVolumeClaim: - claimName: mongo-data - - name: mongo-config - persistentVolumeClaim: - claimName: mongo-config - - name: kafka-data - persistentVolumeClaim: - claimName: kafka-data - - name: redis-data - persistentVolumeClaim: - claimName: redis-data - - name: loki-data - persistentVolumeClaim: - claimName: loki-data - - name: grafana-data - persistentVolumeClaim: - claimName: grafana-data diff --git a/k8s/infra/pvc-infra.yaml b/k8s/infra/pvc-infra.yaml deleted file mode 100644 index f41062a..0000000 --- a/k8s/infra/pvc-infra.yaml +++ /dev/null @@ -1,71 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: mongo-data - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: mongo-config - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 500Mi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: kafka-data - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: redis-data - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: loki-data - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: grafana-data - namespace: simplens -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/k8s/infra/services-infra.yaml b/k8s/infra/services-infra.yaml deleted file mode 100644 index 3854a73..0000000 --- a/k8s/infra/services-infra.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# ============================================================================== -# Internal ClusterIP Services for Intra-Cluster Communication -# Allows all App Pods to reach MongoDB, Kafka, Redis, and Loki by name -# ============================================================================== -apiVersion: v1 -kind: Service -metadata: - name: mongo - namespace: simplens - labels: - app.kubernetes.io/name: mongo -spec: - type: NodePort - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: mongo - port: 27017 - targetPort: 27017 - nodePort: 30717 ---- -apiVersion: v1 -kind: Service -metadata: - name: kafka - namespace: simplens - labels: - app.kubernetes.io/name: kafka -spec: - type: NodePort - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: internal - port: 9093 - targetPort: 9093 - - name: host - port: 9092 - targetPort: 9092 - nodePort: 30092 ---- -apiVersion: v1 -kind: Service -metadata: - name: redis - namespace: simplens - labels: - app.kubernetes.io/name: redis -spec: - type: NodePort - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: redis - port: 6379 - targetPort: 6379 - nodePort: 30379 ---- -apiVersion: v1 -kind: Service -metadata: - name: loki - namespace: simplens - labels: - app.kubernetes.io/name: loki -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: http-loki - port: 3100 - targetPort: 3100 ---- -# ============================================================================== -# Host-Facing NodePort Services for Web Dashboards -# ============================================================================== -apiVersion: v1 -kind: Service -metadata: - name: kafka-ui - namespace: simplens - labels: - app.kubernetes.io/name: kafka-ui -spec: - type: NodePort - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: http-kafkaui - port: 8080 - targetPort: 8080 - nodePort: 30080 ---- -apiVersion: v1 -kind: Service -metadata: - name: grafana - namespace: simplens - labels: - app.kubernetes.io/name: grafana -spec: - type: NodePort - selector: - app.kubernetes.io/name: simplens-infra - ports: - - name: http-grafana - port: 3000 - targetPort: 3000 - nodePort: 30001 diff --git a/k8s/kind-config.yaml b/k8s/kind-config.yaml deleted file mode 100644 index eedba61..0000000 --- a/k8s/kind-config.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# ============================================================================== -# Kind Cluster Configuration with Port Mappings for NodePorts -# Use this configuration when creating a cluster with: -# kind create cluster --name simplens --config k8s/kind-config.yaml -# ============================================================================== -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: - - role: control-plane - extraPortMappings: - # ============================================ - # Infrastructure Services - # ============================================ - - containerPort: 30001 - hostPort: 30001 - listenAddress: "127.0.0.1" # Grafana (Host: http://localhost:30001) - - containerPort: 30080 - hostPort: 30080 - listenAddress: "127.0.0.1" # Kafka-UI (Host: http://localhost:30080) - - containerPort: 30092 - hostPort: 30092 - listenAddress: "127.0.0.1" # Kafka external host listener - - containerPort: 30379 - hostPort: 30379 - listenAddress: "127.0.0.1" # Redis - - containerPort: 30717 - hostPort: 30717 - listenAddress: "127.0.0.1" # MongoDB - - # ============================================ - # App Master (Prod - GHCR Master) - # ============================================ - - containerPort: 30100 - hostPort: 30100 - listenAddress: "127.0.0.1" # Master API (Host: http://localhost:30100) - - containerPort: 30102 - hostPort: 30102 - listenAddress: "127.0.0.1" # Master Dashboard (Host: http://localhost:30102) - - # ============================================ - # App Development (GHCR Dev Branch) - # ============================================ - - containerPort: 30200 - hostPort: 30200 - listenAddress: "127.0.0.1" # Development API (Host: http://localhost:30200) - - containerPort: 30202 - hostPort: 30202 - listenAddress: "127.0.0.1" # Development Dashboard (Host: http://localhost:30202) - - # ============================================ - # App Local (Locally Built Image) - # ============================================ - - containerPort: 30300 - hostPort: 30300 - listenAddress: "127.0.0.1" # Local API (Host: http://localhost:30300) - - containerPort: 30302 - hostPort: 30302 - listenAddress: "127.0.0.1" # Local Dashboard (Host: http://localhost:30302) diff --git a/k8s/scripts/win/build-local.ps1 b/k8s/scripts/win/build-local.ps1 deleted file mode 100644 index 8fd823e..0000000 --- a/k8s/scripts/win/build-local.ps1 +++ /dev/null @@ -1,92 +0,0 @@ -<# -.SYNOPSIS - Builds the local SimpleNS Core and Dashboard Docker images and loads them into Kind (if using Kind). - -.PARAMETER ClusterName - The name of the Kind cluster (default: "simplens"). - -.PARAMETER LoadKind - Switch to load the built images into Kind using 'kind load docker-image'. -#> -param( - [string]$ClusterName = "simplens", - [switch]$LoadKind, - [switch]$NoCache -) - -$ErrorActionPreference = "Stop" -$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path -$K8sDir = Split-Path -Parent (Split-Path -Parent $ScriptDir) -$RootDir = Split-Path -Parent $K8sDir - -$BuildFlags = if ($NoCache) { "--no-cache" } else { "" } - -Write-Host "==========================================" -ForegroundColor Cyan -Write-Host " Building Local SimpleNS Docker Images " -ForegroundColor Cyan -Write-Host "==========================================" -ForegroundColor Cyan - -# 1. Build simplens-core:local -Write-Host "`n[1/2] Building simplens-core:local from root Dockerfile..." -ForegroundColor Yellow -docker build -t simplens-core:local -f "$RootDir/Dockerfile" "$RootDir" $BuildFlags -if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to build simplens-core:local" -} -Write-Host "[OK] simplens-core:local built successfully!" -ForegroundColor Green - -# 2. Build simplens-dashboard:local -Write-Host "`n[2/2] Building simplens-dashboard:local from dashboard/Dockerfile..." -ForegroundColor Yellow -docker build -t simplens-dashboard:local -f "$RootDir/dashboard/Dockerfile" "$RootDir/dashboard" $BuildFlags -if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to build simplens-dashboard:local" -} -Write-Host "[OK] simplens-dashboard:local built successfully!" -ForegroundColor Green - -# 3. Load into Kind if requested or if Kind cluster is detected -$TargetKindCluster = $ClusterName -$ShouldLoadKind = $LoadKind.IsPresent - -if (-not $ShouldLoadKind) { - if (Get-Command kind -ErrorAction SilentlyContinue) { - $rawClusters = (kind get clusters 2>$null) - if ($rawClusters) { - $clusters = ($rawClusters -split "`r?`n") | Where-Object { $_.Trim() -ne "" } | ForEach-Object { $_.Trim() } - - # Check if current kubectl context indicates a kind cluster - $currentContext = (kubectl config current-context 2>$null) - if ($currentContext -and $currentContext -like "kind-*") { - $contextCluster = $currentContext.Substring(5) - if ($clusters -contains $contextCluster) { - $TargetKindCluster = $contextCluster - $ShouldLoadKind = $true - } - } - - if (-not $ShouldLoadKind) { - if ($clusters -contains $ClusterName) { - $TargetKindCluster = $ClusterName - $ShouldLoadKind = $true - } elseif ($clusters.Count -eq 1) { - $TargetKindCluster = $clusters[0] - $ShouldLoadKind = $true - } - } - } - } -} - -if ($ShouldLoadKind) { - Write-Host "`n[Kind] Loading images into Kind cluster '$TargetKindCluster'..." -ForegroundColor Cyan - kind load docker-image simplens-core:local --name $TargetKindCluster - kind load docker-image simplens-dashboard:local --name $TargetKindCluster - Write-Host "[OK] Images loaded into Kind cluster '$TargetKindCluster'!" -ForegroundColor Green -} else { - Write-Host "`nNote: If using Docker Desktop Kubernetes, local Docker images are shared automatically." -ForegroundColor DarkGray - Write-Host "If using Kind, pass -LoadKind or run: kind load docker-image simplens-core:local --name $ClusterName" -ForegroundColor DarkGray -} - -# 4. Clean up dangling images to avoid duplicate : images in Docker -Write-Host "`nCleaning up dangling Docker images..." -ForegroundColor Cyan -docker image prune -f --filter "dangling=true" 2>$null | Out-Null -Write-Host "[OK] Dangling images cleaned up." -ForegroundColor Green - -Write-Host "`nAll images are ready for Kubernetes!" -ForegroundColor Green diff --git a/k8s/scripts/win/deploy-all.ps1 b/k8s/scripts/win/deploy-all.ps1 deleted file mode 100644 index eb5160b..0000000 --- a/k8s/scripts/win/deploy-all.ps1 +++ /dev/null @@ -1,77 +0,0 @@ -<# -.SYNOPSIS - Deploys the complete SimpleNS Kubernetes stack (Infra Pod + App Pods) to the local cluster. - -.PARAMETER ActiveEnv - The initial active app environment to run: 'master', 'development', or 'local' (default: 'local'). -#> -param( - [ValidateSet("master", "development", "local", "all")] - [string]$ActiveEnv = "local" -) - -$ErrorActionPreference = "Stop" -$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path -$K8sDir = Split-Path -Parent (Split-Path -Parent $ScriptDir) -$RootDir = Split-Path -Parent $K8sDir - -Write-Host "==========================================" -ForegroundColor Cyan -Write-Host " Deploying SimpleNS Multi-Env Kubernetes " -ForegroundColor Cyan -Write-Host "==========================================" -ForegroundColor Cyan - -# 1. Namespace -Write-Host "`n[1/7] Applying namespace..." -ForegroundColor Yellow -kubectl apply -f "$K8sDir/base/namespace.yaml" - -# 2. ConfigMap -Write-Host "`n[2/7] Applying simplens configuration ConfigMap..." -ForegroundColor Yellow -kubectl apply -f "$K8sDir/base/configmap-simplens.yaml" - -# 3. Environment Variables Secret (.env injection) -Write-Host "`n[3/7] Injecting .env file into Kubernetes Secret 'app-env'..." -ForegroundColor Yellow -$EnvPath = "$RootDir/.env" -if (-not (Test-Path $EnvPath)) { - Write-Warning ".env file not found at '$EnvPath'. Falling back to '.env.example'." - $EnvPath = "$RootDir/.env.example" -} -# Create or update Secret idempotently using --dry-run -kubectl create secret generic app-env --from-env-file="$EnvPath" --namespace simplens --dry-run=client -o yaml | kubectl apply -f - -Write-Host "[OK] Secret 'app-env' created/updated from $EnvPath" -ForegroundColor Green - -# 4. Persistent Volumes -Write-Host "`n[4/7] Applying persistent volume claims..." -ForegroundColor Yellow -kubectl apply -f "$K8sDir/infra/pvc-infra.yaml" - -# 5. Infrastructure Pod & Services -Write-Host "`n[5/7] Deploying infrastructure pod (Mongo, Kafka, Redis, Loki, Grafana, Kafka-UI)..." -ForegroundColor Yellow -kubectl apply -f "$K8sDir/infra/pod-infra.yaml" -kubectl apply -f "$K8sDir/infra/services-infra.yaml" - -Write-Host "`nWaiting for infrastructure pod to be ready..." -ForegroundColor Cyan -kubectl wait --namespace simplens --for=condition=ready pod -l app.kubernetes.io/name=simplens-infra --timeout=120s - -# 6. App Deployments -Write-Host "`n[6/7] Deploying application environments (Master, Development, Local)..." -ForegroundColor Yellow -kubectl apply -f "$K8sDir/apps/app-master.yaml" -kubectl apply -f "$K8sDir/apps/app-development.yaml" -kubectl apply -f "$K8sDir/apps/app-local.yaml" - -# 7. Set Active Environment -Write-Host "`n[7/7] Activating '$ActiveEnv' environment..." -ForegroundColor Yellow -& "$ScriptDir/switch-env.ps1" -Env $ActiveEnv - -Write-Host "`n==========================================" -ForegroundColor Green -Write-Host " Deployment Complete! " -ForegroundColor Green -Write-Host "==========================================" -ForegroundColor Green -Write-Host "Active App Environment: $ActiveEnv" -ForegroundColor Cyan -Write-Host "`nAccess URLs (on localhost via port-forward):" -ForegroundColor White -Write-Host " - Active Dashboard: http://localhost:3002 (or 30302 for Local)" -ForegroundColor White -Write-Host " - Active API: http://localhost:3000 (or 30300 for Local)" -ForegroundColor White -Write-Host " - Kafka UI: http://localhost:8080" -ForegroundColor White -Write-Host " - Grafana: http://localhost:3001 (admin / admin)" -ForegroundColor White -Write-Host "`nStart port-forwarding by running:" -ForegroundColor DarkGray -Write-Host " ./k8s/scripts/win/port-forward.ps1" -ForegroundColor DarkGray -Write-Host "`nTo switch testing environments, run:" -ForegroundColor DarkGray -Write-Host " ./k8s/scripts/win/switch-env.ps1 -Env master" -ForegroundColor DarkGray -Write-Host " ./k8s/scripts/win/switch-env.ps1 -Env development" -ForegroundColor DarkGray -Write-Host " ./k8s/scripts/win/switch-env.ps1 -Env local" -ForegroundColor DarkGray diff --git a/k8s/scripts/win/port-forward.ps1 b/k8s/scripts/win/port-forward.ps1 deleted file mode 100644 index ad43ce2..0000000 --- a/k8s/scripts/win/port-forward.ps1 +++ /dev/null @@ -1,154 +0,0 @@ -<# -.SYNOPSIS - Tunnels ports from the Kubernetes cluster to your Windows host machine (localhost). - Supports forwarding for a specific environment OR all active environments simultaneously. - -.PARAMETER Env - The app environment to forward: 'all', 'local', 'master', or 'development' (default: 'all'). -#> -param( - [ValidateSet("all", "local", "master", "development")] - [string]$Env = "all" -) - -$Namespace = "simplens" - -Write-Host "==========================================" -ForegroundColor Cyan -Write-Host " SimpleNS Multi-Env Port Forwarding " -ForegroundColor Cyan -Write-Host "==========================================" -ForegroundColor Cyan - -# Stop any existing kubectl port-forward processes reliably across PowerShell versions -Get-CimInstance Win32_Process -Filter "Name = 'kubectl.exe'" -ErrorAction SilentlyContinue | - Where-Object { $_.CommandLine -like "*port-forward*" } | - ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } - -function Test-PortInUse([int]$port) { - try { - $tcp = New-Object System.Net.Sockets.TcpClient - $iar = $tcp.BeginConnect("127.0.0.1", $port, $null, $null) - $success = $iar.AsyncWaitHandle.WaitOne(200, $false) - if ($success) { - $tcp.EndConnect($iar) - $tcp.Close() - return $true - } - $tcp.Close() - return $false - } catch { - return $false - } -} - -$jobs = @() - -# 1. Forward Kafka-UI (8080) -if (-not (Test-PortInUse 8080)) { - $jobs += Start-Job -ScriptBlock { - param($ns) - while ($true) { - kubectl port-forward -n $ns svc/kafka-ui 8080:8080 2>$null - Start-Sleep -Milliseconds 800 - } - } -ArgumentList $Namespace -} - -# 2. Forward Grafana (3001 -> 3000) -if (-not (Test-PortInUse 3001)) { - $jobs += Start-Job -ScriptBlock { - param($ns) - while ($true) { - kubectl port-forward -n $ns svc/grafana 3001:3000 2>$null - Start-Sleep -Milliseconds 800 - } - } -ArgumentList $Namespace -} - -# Helper function to check if a deployment is running -function Is-Running($deploymentName) { - $replicas = (kubectl get deployment $deploymentName -n $Namespace -o jsonpath='{.spec.replicas}' 2>$null) - return ($replicas -and [int]$replicas -gt 0) -} - -# 3. Forward Local Environment (Ports 3000/3002 and NodePorts 30300/30302) -if (($Env -eq "all" -or $Env -eq "local") -and (Is-Running "app-local")) { - $localPorts = @() - if (-not (Test-PortInUse 3000)) { $localPorts += "3000:3000" } - if (-not (Test-PortInUse 3002)) { $localPorts += "3002:3002" } - if (-not (Test-PortInUse 30300)) { $localPorts += "30300:3000" } - if (-not (Test-PortInUse 30302)) { $localPorts += "30302:3002" } - - if ($localPorts.Count -gt 0) { - $pStr = $localPorts -join " " - $jobs += Start-Job -ScriptBlock { - param($ns, $p) - while ($true) { - Invoke-Expression "kubectl port-forward -n $ns svc/app-local $p 2>`$null" - Start-Sleep -Milliseconds 800 - } - } -ArgumentList $Namespace, $pStr - } - Write-Host "[+] Local App Access (Auto-reconnecting):" -ForegroundColor Green - Write-Host " - Dashboard: http://localhost:3002 (NodePort: http://localhost:30302)" -ForegroundColor White - Write-Host " - API: http://localhost:3000 (NodePort: http://localhost:30300)" -ForegroundColor White -} - -# 4. Forward Master Environment (Ports 30100/30102) -if (($Env -eq "all" -or $Env -eq "master") -and (Is-Running "app-master")) { - $masterPorts = @() - if (-not (Test-PortInUse 30100)) { $masterPorts += "30100:3000" } - if (-not (Test-PortInUse 30102)) { $masterPorts += "30102:3002" } - - if ($masterPorts.Count -gt 0) { - $pStr = $masterPorts -join " " - $jobs += Start-Job -ScriptBlock { - param($ns, $p) - while ($true) { - Invoke-Expression "kubectl port-forward -n $ns svc/app-master $p 2>`$null" - Start-Sleep -Milliseconds 800 - } - } -ArgumentList $Namespace, $pStr - } - Write-Host "[+] Master (Prod) App Access (Auto-reconnecting):" -ForegroundColor Green - Write-Host " - Dashboard: http://localhost:30102" -ForegroundColor White - Write-Host " - API: http://localhost:30100" -ForegroundColor White -} - -# 5. Forward Development Branch Environment (Ports 30200/30202) -if (($Env -eq "all" -or $Env -eq "development") -and (Is-Running "app-development")) { - $devPorts = @() - if (-not (Test-PortInUse 30200)) { $devPorts += "30200:3000" } - if (-not (Test-PortInUse 30202)) { $devPorts += "30202:3002" } - - if ($devPorts.Count -gt 0) { - $pStr = $devPorts -join " " - $jobs += Start-Job -ScriptBlock { - param($ns, $p) - while ($true) { - Invoke-Expression "kubectl port-forward -n $ns svc/app-development $p 2>`$null" - Start-Sleep -Milliseconds 800 - } - } -ArgumentList $Namespace, $pStr - } - Write-Host "[+] Development Branch App Access (Auto-reconnecting):" -ForegroundColor Green - Write-Host " - Dashboard: http://localhost:30202" -ForegroundColor White - Write-Host " - API: http://localhost:30200" -ForegroundColor White -} - -Start-Sleep -Seconds 2 - -Write-Host "`n[+] Infrastructure Services:" -ForegroundColor Green -Write-Host " - Kafka UI: http://localhost:8080" -ForegroundColor White -Write-Host " - Grafana: http://localhost:3001 (admin / admin)" -ForegroundColor White - -Write-Host "`n[OK] Port forwarding is active! Press Ctrl+C to close all tunnels.`n" -ForegroundColor Cyan - -try { - while ($true) { - Start-Sleep -Seconds 1 - } -} finally { - Write-Host "`nClosing port tunnels..." -ForegroundColor Yellow - $jobs | Stop-Job -ErrorAction SilentlyContinue - $jobs | Remove-Job -ErrorAction SilentlyContinue - Write-Host "[OK] Tunnels closed." -ForegroundColor Green -} diff --git a/k8s/scripts/win/switch-env.ps1 b/k8s/scripts/win/switch-env.ps1 deleted file mode 100644 index 1649c1b..0000000 --- a/k8s/scripts/win/switch-env.ps1 +++ /dev/null @@ -1,155 +0,0 @@ -<# -.SYNOPSIS - Switches the active SimpleNS application environment (Master, Development, Local, or All). - Ensures that multiple environments do not contend on shared Kafka consumer groups. - -.PARAMETER Env - The environment to activate: 'master', 'development', 'local', 'all', or 'status'. - -.PARAMETER PullLatest - If specified, pulls the latest images from GHCR (for master/development) or rebuilds local images (for local) before running. -#> -param( - [ValidateSet("master", "development", "local", "all", "status")] - [string]$Env = "status", - - [switch]$PullLatest, - [switch]$NoCache -) - -$Namespace = "simplens" -$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path - -function Show-Status { - Write-Host "`n--- Current SimpleNS Pod Status ---" -ForegroundColor Cyan - kubectl get pods -n $Namespace -o wide - Write-Host "`n--- App Deployments Replica Counts ---" -ForegroundColor Cyan - kubectl get deployments -n $Namespace -l "app.kubernetes.io/name in (app-master, app-development, app-local)" -} - -function Sync-ToKind { - param([string[]]$Images) - if (Get-Command kind -ErrorAction SilentlyContinue) { - $rawClusters = (kind get clusters 2>$null) - if ($rawClusters) { - $clusters = ($rawClusters -split "`r?`n") | Where-Object { $_.Trim() -ne "" } | ForEach-Object { $_.Trim() } - $targetCluster = $null - $currentContext = (kubectl config current-context 2>$null) - if ($currentContext -and $currentContext -like "kind-*") { - $contextCluster = $currentContext.Substring(5) - if ($clusters -contains $contextCluster) { - $targetCluster = $contextCluster - } - } - if (-not $targetCluster) { - if ($clusters -contains "simplens") { - $targetCluster = "simplens" - } elseif ($clusters.Count -eq 1) { - $targetCluster = $clusters[0] - } - } - if ($targetCluster) { - foreach ($img in $Images) { - Write-Host "[Kind] Loading image '$img' into cluster '$targetCluster'..." -ForegroundColor Cyan - kind load docker-image $img --name $targetCluster - } - } - } - } -} - -function Pull-DevelopmentImages { - Write-Host "`n[Pull] Pulling latest development images from GHCR..." -ForegroundColor Cyan - docker pull ghcr.io/simplenotificationsystem/simplens-core:development - docker pull ghcr.io/simplenotificationsystem/simplens-dashboard:development - Write-Host "[OK] Latest development images pulled." -ForegroundColor Green - Sync-ToKind @("ghcr.io/simplenotificationsystem/simplens-core:development", "ghcr.io/simplenotificationsystem/simplens-dashboard:development") -} - -function Pull-MasterImages { - Write-Host "`n[Pull] Pulling latest master images from GHCR..." -ForegroundColor Cyan - docker pull ghcr.io/simplenotificationsystem/simplens-core:latest - docker pull ghcr.io/simplenotificationsystem/simplens-dashboard:latest - Write-Host "[OK] Latest master images pulled." -ForegroundColor Green - Sync-ToKind @("ghcr.io/simplenotificationsystem/simplens-core:latest", "ghcr.io/simplenotificationsystem/simplens-dashboard:latest") -} - -switch ($Env) { - "master" { - if ($PullLatest) { - Pull-MasterImages - } - Write-Host "Activating 'master' environment (GHCR master branch)..." -ForegroundColor Yellow - kubectl scale deployment app-master -n $Namespace --replicas=1 - kubectl scale deployment app-development -n $Namespace --replicas=0 - kubectl scale deployment app-local -n $Namespace --replicas=0 - if ($PullLatest) { - kubectl rollout restart deployment app-master -n $Namespace - Write-Host "Waiting for app-master rollout to complete..." -ForegroundColor Cyan - kubectl rollout status deployment app-master -n $Namespace --timeout=120s - } - Write-Host "[OK] Switched to 'master' environment!" -ForegroundColor Green - Write-Host " API: http://localhost:30100" -ForegroundColor White - Write-Host " Dashboard: http://localhost:30102" -ForegroundColor White - } - "development" { - if ($PullLatest) { - Pull-DevelopmentImages - } - Write-Host "Activating 'development' environment (GHCR dev branch)..." -ForegroundColor Yellow - kubectl scale deployment app-master -n $Namespace --replicas=0 - kubectl scale deployment app-development -n $Namespace --replicas=1 - kubectl scale deployment app-local -n $Namespace --replicas=0 - if ($PullLatest) { - kubectl rollout restart deployment app-development -n $Namespace - Write-Host "Waiting for app-development rollout to complete..." -ForegroundColor Cyan - kubectl rollout status deployment app-development -n $Namespace --timeout=120s - } - Write-Host "[OK] Switched to 'development' environment!" -ForegroundColor Green - Write-Host " API: http://localhost:30200" -ForegroundColor White - Write-Host " Dashboard: http://localhost:30202" -ForegroundColor White - } - "local" { - if ($PullLatest) { - Write-Host "`n[Build] Rebuilding local images..." -ForegroundColor Cyan - $buildArgs = @() - if ($NoCache) { $buildArgs += "-NoCache" } - & "$ScriptDir/build-local.ps1" @buildArgs - } - Write-Host "Activating 'local' environment (locally built image)..." -ForegroundColor Yellow - kubectl scale deployment app-master -n $Namespace --replicas=0 - kubectl scale deployment app-development -n $Namespace --replicas=0 - kubectl scale deployment app-local -n $Namespace --replicas=1 - if ($PullLatest) { - Write-Host "Restarting deployment app-local..." -ForegroundColor Cyan - kubectl rollout restart deployment app-local -n $Namespace - Write-Host "Waiting for app-local rollout to complete..." -ForegroundColor Cyan - kubectl rollout status deployment app-local -n $Namespace --timeout=120s - } - Write-Host "[OK] Switched to 'local' environment!" -ForegroundColor Green - Write-Host " API: http://localhost:30300" -ForegroundColor White - Write-Host " Dashboard: http://localhost:30302" -ForegroundColor White - } - "all" { - if ($PullLatest) { - Pull-MasterImages - Pull-DevelopmentImages - } - Write-Host "Scaling all environments to 1 replica..." -ForegroundColor Yellow - Write-Warning "Notice: All 3 app environments share the same Kafka broker. Because consumer group IDs are shared, consumers will balance partitions across environments." - kubectl scale deployment app-master -n $Namespace --replicas=1 - kubectl scale deployment app-development -n $Namespace --replicas=1 - kubectl scale deployment app-local -n $Namespace --replicas=1 - if ($PullLatest) { - kubectl rollout restart deployment app-master -n $Namespace - kubectl rollout restart deployment app-development -n $Namespace - } - Write-Host "[OK] All environments scaled to 1!" -ForegroundColor Green - } - "status" { - Show-Status - return - } -} - -Show-Status diff --git a/package-lock.json b/package-lock.json index 6d43e14..bcc3040 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "simplens-core", - "version": "1.3.0", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "simplens-core", - "version": "1.3.0", + "version": "1.4.0", "license": "ISC", "dependencies": { "axios": "^1.13.2", @@ -16,6 +16,7 @@ "express-rate-limit": "^8.2.1", "helmet": "^8.1.0", "ioredis": "^5.8.2", + "jsonwebtoken": "^9.0.3", "kafkajs": "^2.2.4", "mongoose": "^9.0.0", "uuid": "^13.0.0", @@ -31,6 +32,7 @@ "@types/express-rate-limit": "^5.1.3", "@types/helmet": "^0.0.48", "@types/ioredis": "^4.28.10", + "@types/jsonwebtoken": "^9.0.10", "@types/kafkajs": "^1.8.2", "@types/mongoose": "^5.11.96", "@types/node": "^24.10.1", @@ -2635,6 +2637,17 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/jsonwebtoken": { + "version": "9.0.10", + "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.10.tgz", + "integrity": "sha512-asx5hIG9Qmf/1oStypjanR7iKTv0gXQ1Ov/jfrX6kS/EO0OFni8orbmGCn0672NHR3kXHwpAwR+B368ZGN/2rA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ms": "*", + "@types/node": "*" + } + }, "node_modules/@types/kafkajs": { "version": "1.8.2", "resolved": "https://registry.npmjs.org/@types/kafkajs/-/kafkajs-1.8.2.tgz", @@ -2662,6 +2675,13 @@ "mongoose": "*" } }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "24.10.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", @@ -3644,6 +3664,12 @@ "node": ">= 0.4.0" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/bundle-name": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", @@ -4359,6 +4385,15 @@ "node": ">= 0.4" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/eciesjs": { "version": "0.4.17", "resolved": "https://registry.npmjs.org/eciesjs/-/eciesjs-0.4.17.tgz", @@ -6265,6 +6300,49 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jsonwebtoken": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", + "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", + "license": "MIT", + "dependencies": { + "jws": "^4.0.1", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/kafkajs": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-2.2.4.tgz", @@ -6610,12 +6688,54 @@ "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", "license": "MIT" }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", + "license": "MIT" + }, "node_modules/lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", "license": "MIT" }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", + "license": "MIT" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", + "license": "MIT" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "license": "MIT" + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", + "license": "MIT" + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "license": "MIT" + }, "node_modules/log-symbols": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-6.0.0.tgz", @@ -8300,7 +8420,6 @@ "version": "7.7.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" diff --git a/package.json b/package.json index 765c37e..6819c22 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "simplens-core", - "version": "1.3.0", + "version": "1.4.0", "private": true, "description": "Open-source plugin-based notification orchestration engine for developers who value control ", "main": "dist/api/server.js", @@ -47,6 +47,7 @@ "@types/express-rate-limit": "^5.1.3", "@types/helmet": "^0.0.48", "@types/ioredis": "^4.28.10", + "@types/jsonwebtoken": "^9.0.10", "@types/kafkajs": "^1.8.2", "@types/mongoose": "^5.11.96", "@types/node": "^24.10.1", @@ -74,6 +75,7 @@ "express-rate-limit": "^8.2.1", "helmet": "^8.1.0", "ioredis": "^5.8.2", + "jsonwebtoken": "^9.0.3", "kafkajs": "^2.2.4", "mongoose": "^9.0.0", "uuid": "^13.0.0", diff --git a/packages/mcp-server/README.md b/packages/mcp-server/README.md index 644f1b3..3241b12 100644 --- a/packages/mcp-server/README.md +++ b/packages/mcp-server/README.md @@ -8,7 +8,7 @@ A Model Context Protocol (MCP) server for the SimpleNS notification orchestratio - **Streamable HTTP Transport**: Connect remote or local MCP clients via HTTP (`/mcp`). - **Stdio Transport**: Local execution via command line (`--stdio`). -- **Full Tool Suite (29 Tools)**: Comprehensive coverage for notifications, templates, alerts, logs, metrics, and channel configuration. +- **Full Tool Suite (54 Tools)**: Comprehensive coverage for notifications, templates, alerts, logs, metrics, channel routing, providers, rate limits, dynamic settings, plugin lifecycle, and admin configuration. --- @@ -87,7 +87,7 @@ Add to your MCP Client configuration: --- -## Complete Tools Reference (29 Tools) +## Complete Tools Reference (54 Tools) ### 1. Notification Dispatch | Tool | Description | @@ -145,3 +145,49 @@ Add to your MCP Client configuration: | `delete_admin_channel` | Delete an admin channel configuration. | | `test_admin_channel` | Execute a live connection and credential test for an admin channel. | | `validate_admin_channel_config` | Validate configuration parameters against provider requirements before saving. | + +### 8. Channel Routing & Kafka Partition Scaling +| Tool | Description | +| :--- | :--- | +| `list_channel_routings` | List all configured channel routings showing default provider, fallback cascade, and partition counts. | +| `get_channel_routing` | Retrieve routing configuration and fallback chain for a specific channel (e.g. email, sms). | +| `set_channel_routing` | Configure default provider, ordered cascading fallback providers, and Kafka partitions for a channel. | +| `delete_channel_routing` | Remove routing rules for a channel. | + +### 9. Provider Management & Real-Time Rate Limiting +| Tool | Description | +| :--- | :--- | +| `list_providers` | List all configured provider instances with their enabled status and options. | +| `get_provider` | Retrieve detailed configuration and status for a single provider instance (supports optional decrypted view). | +| `create_provider` | Instantiate and register a new provider instance with credentials and rate limit options. | +| `update_provider` | Update credentials, options, or toggle enabled state of an existing provider. | +| `delete_provider` | Remove a provider instance (verifies no active channel routings depend on it). | +| `test_provider` | Test connection and credentials for an existing provider or unpersisted configuration. | +| `get_providers_rate_limits` | Real-time token bucket capacity, refill rates, and metrics across all providers. | +| `get_provider_rate_limit` | Real-time rate limit status and remaining token capacity for a specific provider instance. | +| `reset_provider_rate_limit` | Manually reset rate limiter tokens and telemetry counters for a specific provider in Redis. | + +### 10. Dynamic Operational Settings +| Tool | Description | +| :--- | :--- | +| `get_operational_settings` | Retrieve active operational settings (worker concurrency, retry limits, polling intervals, DLQ thresholds). | +| `update_operational_settings` | Hot-update operational settings cluster-wide via Redis pub/sub with zero service restarts. | +| `reset_operational_settings` | Reset all dynamic operational configuration parameters to system defaults. | + +### 11. Plugin Lifecycle & NPM Registry Management +| Tool | Description | +| :--- | :--- | +| `list_installed_plugins` | List all installed npm plugin packages stored in MongoDB with version and installation timestamp. | +| `get_plugin_catalog` | Browse available plugins from the public catalog by category (`official` or `community`). | +| `install_plugin` | Download, validate, and hot-load an npm provider plugin package with optional private auth. | +| `change_plugin_version` | Upgrade or downgrade an installed plugin to a specified semver version. | +| `uninstall_plugin` | Uninstall an installed plugin package and clean up associated resources. | +| `get_npm_auth` | Check configured npm registry authentication status and masked tokens. | +| `save_npm_auth` | Save private npm registry credentials (auth token, registry URL, optional scope). | +| `delete_npm_auth` | Remove saved npm registry authentication credentials. | + +### 12. Admin Setup Status +| Tool | Description | +| :--- | :--- | +| `get_admin_auth_status` | Check whether SimpleNS administrator credentials have been configured or require initial setup. | + diff --git a/packages/mcp-server/package-lock.json b/packages/mcp-server/package-lock.json index 68eb195..691bb63 100644 --- a/packages/mcp-server/package-lock.json +++ b/packages/mcp-server/package-lock.json @@ -1,12 +1,12 @@ { "name": "@simplens/mcp", - "version": "1.1.2", + "version": "1.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@simplens/mcp", - "version": "1.1.2", + "version": "1.2.0", "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "^1.12.1", diff --git a/packages/mcp-server/package.json b/packages/mcp-server/package.json index dce3fae..486d496 100644 --- a/packages/mcp-server/package.json +++ b/packages/mcp-server/package.json @@ -1,6 +1,6 @@ { "name": "@simplens/mcp", - "version": "1.1.2", + "version": "1.2.0", "description": "Remote MCP server for SimpleNS notification orchestration engine", "type": "module", "main": "dist/index.js", diff --git a/packages/mcp-server/src/api-client.ts b/packages/mcp-server/src/api-client.ts index 9b76265..fde3ff0 100644 --- a/packages/mcp-server/src/api-client.ts +++ b/packages/mcp-server/src/api-client.ts @@ -351,6 +351,256 @@ export class ApiClient { headers: this.authHeader, }); } + + // ============================================================================ + // CHANNEL ROUTING + // ============================================================================ + + /** GET /api/channels/routing - List all channel routings */ + async listChannelRoutings(): Promise { + return request(this.baseUrl, 'api/channels/routing', { + headers: this.authHeader, + }); + } + + /** GET /api/channels/routing/:channel - Get routing for a specific channel */ + async getChannelRouting(channel: string): Promise { + return request(this.baseUrl, `api/channels/routing/${encodeURIComponent(channel)}`, { + headers: this.authHeader, + }); + } + + /** PUT /api/channels/routing/:channel - Set or update routing for a channel */ + async setChannelRouting(channel: string, payload: { + default_provider_id: string; + fallback_provider_ids?: string[]; + partitions?: number; + }): Promise { + return request(this.baseUrl, `api/channels/routing/${encodeURIComponent(channel)}`, { + method: 'PUT', + headers: this.authHeader, + body: payload, + }); + } + + /** DELETE /api/channels/routing/:channel - Delete routing for a channel */ + async deleteChannelRouting(channel: string): Promise { + return request(this.baseUrl, `api/channels/routing/${encodeURIComponent(channel)}`, { + method: 'DELETE', + headers: this.authHeader, + }); + } + + // ============================================================================ + // PROVIDERS & RATE LIMITS + // ============================================================================ + + /** GET /api/providers - List configured providers */ + async listProviders(): Promise { + return request(this.baseUrl, 'api/providers', { + headers: this.authHeader, + }); + } + + /** GET /api/providers/:id - Get single provider by ID */ + async getProvider(id: string, includeDecrypted = false): Promise { + return request(this.baseUrl, `api/providers/${encodeURIComponent(id)}`, { + headers: this.authHeader, + params: includeDecrypted ? { include_decrypted: 'true' } : undefined, + }); + } + + /** POST /api/providers - Create a new provider */ + async createProvider(payload: { + id: string; + plugin_name: string; + credentials: Record; + options?: Record; + enabled?: boolean; + }): Promise { + return request(this.baseUrl, 'api/providers', { + method: 'POST', + headers: this.authHeader, + body: payload, + }); + } + + /** PUT /api/providers/:id - Update an existing provider */ + async updateProvider(id: string, payload: { + credentials?: Record; + options?: Record; + enabled?: boolean; + }): Promise { + return request(this.baseUrl, `api/providers/${encodeURIComponent(id)}`, { + method: 'PUT', + headers: this.authHeader, + body: payload, + }); + } + + /** DELETE /api/providers/:id - Delete a provider instance */ + async deleteProvider(id: string): Promise { + return request(this.baseUrl, `api/providers/${encodeURIComponent(id)}`, { + method: 'DELETE', + headers: this.authHeader, + }); + } + + /** POST /api/providers/test - Test provider connection/credentials */ + async testProvider(payload: { + provider_id?: string; + plugin_name?: string; + credentials?: Record; + options?: Record; + }): Promise { + return request(this.baseUrl, 'api/providers/test', { + method: 'POST', + headers: this.authHeader, + body: payload, + }); + } + + /** GET /api/providers/rate-limits - Get rate limits for all providers */ + async getProvidersRateLimits(): Promise { + return request(this.baseUrl, 'api/providers/rate-limits', { + headers: this.authHeader, + }); + } + + /** GET /api/providers/:id/rate-limit - Get rate limit status for a provider */ + async getProviderRateLimit(id: string): Promise { + return request(this.baseUrl, `api/providers/${encodeURIComponent(id)}/rate-limit`, { + headers: this.authHeader, + }); + } + + /** POST /api/providers/:id/rate-limit/reset - Reset rate limit for a provider */ + async resetProviderRateLimit(id: string): Promise { + return request(this.baseUrl, `api/providers/${encodeURIComponent(id)}/rate-limit/reset`, { + method: 'POST', + headers: this.authHeader, + }); + } + + // ============================================================================ + // OPERATIONAL DYNAMIC SETTINGS + // ============================================================================ + + /** GET /api/settings - Get operational settings */ + async getOperationalSettings(): Promise { + return request(this.baseUrl, 'api/settings', { + headers: this.authHeader, + }); + } + + /** PUT /api/settings - Update operational settings */ + async updateOperationalSettings(payload: Record): Promise { + return request(this.baseUrl, 'api/settings', { + method: 'PUT', + headers: this.authHeader, + body: payload, + }); + } + + /** POST /api/settings/reset - Reset operational settings to defaults */ + async resetOperationalSettings(): Promise { + return request(this.baseUrl, 'api/settings/reset', { + method: 'POST', + headers: this.authHeader, + }); + } + + // ============================================================================ + // EXTENDED PLUGINS & NPM AUTH + // ============================================================================ + + /** GET /api/plugins/installed - List installed plugins in MongoDB */ + async listInstalledPlugins(): Promise { + return request(this.baseUrl, 'api/plugins/installed', { + headers: this.authHeader, + }); + } + + /** GET /api/plugins/catalog/:category - Get plugin catalog */ + async getPluginCatalog(category: 'official' | 'community'): Promise { + return request(this.baseUrl, `api/plugins/catalog/${category}`, { + headers: this.authHeader, + }); + } + + /** POST /api/plugins/install - Install npm plugin package */ + async installPlugin(payload: { + package: string; + version?: string; + auth?: { token?: string; registry_url?: string }; + }): Promise { + return request(this.baseUrl, 'api/plugins/install', { + method: 'POST', + headers: this.authHeader, + body: payload, + }); + } + + /** PUT /api/plugins/version - Change version of installed plugin */ + async changePluginVersion(payload: { + package: string; + version: string; + }): Promise { + return request(this.baseUrl, 'api/plugins/version', { + method: 'PUT', + headers: this.authHeader, + body: payload, + }); + } + + /** DELETE /api/plugins - Uninstall plugin */ + async uninstallPlugin(packageName: string): Promise { + return request(this.baseUrl, 'api/plugins', { + method: 'DELETE', + headers: this.authHeader, + params: { package: packageName }, + }); + } + + /** GET /api/plugins/npm-auth - Get npm auth status */ + async getNpmAuth(): Promise { + return request(this.baseUrl, 'api/plugins/npm-auth', { + headers: this.authHeader, + }); + } + + /** POST /api/plugins/npm-auth - Save npm auth credentials */ + async saveNpmAuth(payload: { + token: string; + registry_url?: string; + scope?: string; + }): Promise { + return request(this.baseUrl, 'api/plugins/npm-auth', { + method: 'POST', + headers: this.authHeader, + body: payload, + }); + } + + /** DELETE /api/plugins/npm-auth - Delete npm auth configuration */ + async deleteNpmAuth(target?: string): Promise { + return request(this.baseUrl, 'api/plugins/npm-auth', { + method: 'DELETE', + headers: this.authHeader, + params: target ? { id: target } : undefined, + }); + } + + // ============================================================================ + // ADMIN AUTH STATUS + // ============================================================================ + + /** GET /api/admin/auth/status - Check if admin credentials are configured */ + async getAdminAuthStatus(): Promise { + return request(this.baseUrl, 'api/admin/auth/status', { + headers: this.authHeader, + }); + } } diff --git a/packages/mcp-server/src/tools/admin-auth.ts b/packages/mcp-server/src/tools/admin-auth.ts new file mode 100644 index 0000000..c70e30c --- /dev/null +++ b/packages/mcp-server/src/tools/admin-auth.ts @@ -0,0 +1,29 @@ +/** + * Tools: Admin Authentication & Setup Status + */ + +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { UserCredentials } from '../auth.js'; +import { ApiClient } from '../api-client.js'; +import { formatApiResponse, formatToolError } from './response.js'; + +export function registerAdminAuthTools(server: McpServer, getCredentials: () => UserCredentials) { + // 1. get_admin_auth_status + server.registerTool( + 'get_admin_auth_status', + { + description: 'Check whether SimpleNS administrator credentials have been configured, or whether the system is pending initial dashboard onboarding.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getAdminAuthStatus(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to check admin auth status', error); + } + } + ); +} diff --git a/packages/mcp-server/src/tools/channel-routing.ts b/packages/mcp-server/src/tools/channel-routing.ts new file mode 100644 index 0000000..14992d3 --- /dev/null +++ b/packages/mcp-server/src/tools/channel-routing.ts @@ -0,0 +1,97 @@ +/** + * Tools: Channel Routing Configuration + */ + +import { z } from 'zod'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { UserCredentials } from '../auth.js'; +import { ApiClient } from '../api-client.js'; +import { formatApiResponse, formatToolError } from './response.js'; + +export function registerChannelRoutingTools(server: McpServer, getCredentials: () => UserCredentials) { + // 1. list_channel_routings + server.registerTool( + 'list_channel_routings', + { + description: 'List all configured channel routings, showing the primary default provider, ordered cascading fallback providers, and Kafka partitions assigned to each channel (e.g. email, sms, whatsapp).', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.listChannelRoutings(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to list channel routings', error); + } + } + ); + + // 2. get_channel_routing + server.registerTool( + 'get_channel_routing', + { + description: 'Get routing configuration for a specific communication channel (e.g. "email", "sms", "whatsapp"). Returns default provider ID, fallback provider chain, and partition count.', + inputSchema: { + channel: z.string().describe('Channel identifier (e.g. "email", "sms", "whatsapp", "inapp")'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getChannelRouting(params.channel); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to get routing for channel '${params.channel}'`, error); + } + } + ); + + // 3. set_channel_routing + server.registerTool( + 'set_channel_routing', + { + description: 'Set or update provider routing for a channel. Specifies the default provider to attempt first, optional ordered fallback providers if the primary fails or rate limits, and optional Kafka partitions for scaling throughput.', + inputSchema: { + channel: z.string().describe('Channel identifier (e.g. "email", "sms", "whatsapp")'), + default_provider_id: z.string().describe('The primary provider ID configured in SimpleNS to route messages through'), + fallback_provider_ids: z.array(z.string()).optional().describe('Ordered list of fallback provider IDs to cascade to if the default provider fails or is rate limited'), + partitions: z.number().int().min(1).optional().describe('Number of Kafka topic partitions for this channel (can only be scaled up, not down)'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const { channel, ...payload } = params; + const result = await client.setChannelRouting(channel, payload); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to set routing for channel '${params.channel}'`, error); + } + } + ); + + // 4. delete_channel_routing + server.registerTool( + 'delete_channel_routing', + { + description: 'Delete routing configuration for a channel. Messages sent to this channel will fail until routing is re-configured.', + inputSchema: { + channel: z.string().describe('Channel identifier to delete routing for (e.g. "email", "sms")'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.deleteChannelRouting(params.channel); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to delete routing for channel '${params.channel}'`, error); + } + } + ); +} diff --git a/packages/mcp-server/src/tools/index.ts b/packages/mcp-server/src/tools/index.ts index 12e6367..1365c1f 100644 --- a/packages/mcp-server/src/tools/index.ts +++ b/packages/mcp-server/src/tools/index.ts @@ -20,6 +20,11 @@ import { registerNotificationsManagementTools } from './notifications-management import { registerResolveAlertsRetryTools } from './resolve-alerts-retry.js'; import { registerDashboardTools } from './dashboard.js'; import { registerAdminChannelsTools } from './admin-channels.js'; +import { registerChannelRoutingTools } from './channel-routing.js'; +import { registerProviderTools } from './providers.js'; +import { registerSettingsTools } from './settings.js'; +import { registerPluginsManagementTools } from './plugins-management.js'; +import { registerAdminAuthTools } from './admin-auth.js'; export function registerAllTools(server: McpServer, getCredentials: () => UserCredentials) { registerSendNotification(server, getCredentials); @@ -35,4 +40,9 @@ export function registerAllTools(server: McpServer, getCredentials: () => UserCr registerResolveAlertsRetryTools(server, getCredentials); registerDashboardTools(server, getCredentials); registerAdminChannelsTools(server, getCredentials); + registerChannelRoutingTools(server, getCredentials); + registerProviderTools(server, getCredentials); + registerSettingsTools(server, getCredentials); + registerPluginsManagementTools(server, getCredentials); + registerAdminAuthTools(server, getCredentials); } diff --git a/packages/mcp-server/src/tools/plugins-management.ts b/packages/mcp-server/src/tools/plugins-management.ts new file mode 100644 index 0000000..e427b61 --- /dev/null +++ b/packages/mcp-server/src/tools/plugins-management.ts @@ -0,0 +1,183 @@ +/** + * Tools: Extended Plugin Lifecycle & NPM Registry Management + */ + +import { z } from 'zod'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { UserCredentials } from '../auth.js'; +import { ApiClient } from '../api-client.js'; +import { formatApiResponse, formatToolError } from './response.js'; + +export function registerPluginsManagementTools(server: McpServer, getCredentials: () => UserCredentials) { + // 1. list_installed_plugins + server.registerTool( + 'list_installed_plugins', + { + description: 'List all installed npm plugin packages persisted in MongoDB with their installation timestamp, version, and status.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.listInstalledPlugins(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to list installed plugins', error); + } + } + ); + + // 2. get_plugin_catalog + server.registerTool( + 'get_plugin_catalog', + { + description: 'Browse the public SimpleNS plugin catalog to discover available provider plugins by category ("official" or "community").', + inputSchema: { + category: z.enum(['official', 'community']).describe('Plugin category: "official" for verified first-party plugins or "community" for community plugins'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getPluginCatalog(params.category); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to fetch ${params.category} plugin catalog`, error); + } + } + ); + + // 3. install_plugin + server.registerTool( + 'install_plugin', + { + description: 'Install an npm plugin package into SimpleNS. The core engine downloads, validates, and hot-loads the plugin across the cluster.', + inputSchema: { + package: z.string().describe('The npm package name (e.g. "@simplens/smtp", "@simplens/resend")'), + version: z.string().optional().describe('Optional specific semver version or tag (default: "latest")'), + auth: z.object({ + token: z.string().optional().describe('Authentication token for private npm registry or package'), + registry_url: z.string().optional().describe('Custom registry URL if not installing from the default npm registry'), + }).optional().describe('Optional authentication details for private registries or scoped packages'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.installPlugin(params); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to install plugin '${params.package}'`, error); + } + } + ); + + // 4. change_plugin_version + server.registerTool( + 'change_plugin_version', + { + description: 'Upgrade or downgrade an installed plugin to a specific npm version.', + inputSchema: { + package: z.string().describe('The installed npm package name'), + version: z.string().describe('The target semver version to upgrade or downgrade to (e.g. "1.2.0")'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.changePluginVersion(params); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to change version of plugin '${params.package}'`, error); + } + } + ); + + // 5. uninstall_plugin + server.registerTool( + 'uninstall_plugin', + { + description: 'Uninstall a plugin package from SimpleNS. Fails if existing providers depend on this plugin.', + inputSchema: { + package: z.string().describe('The npm package name to uninstall (e.g. "@simplens/resend")'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.uninstallPlugin(params.package); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to uninstall plugin '${params.package}'`, error); + } + } + ); + + // 6. get_npm_auth + server.registerTool( + 'get_npm_auth', + { + description: 'Retrieve configured npm registry authentication status and masked tokens.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getNpmAuth(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to retrieve npm auth configuration', error); + } + } + ); + + // 7. save_npm_auth + server.registerTool( + 'save_npm_auth', + { + description: 'Configure and save npm authentication credentials (auth token and optional registry URL or scope) for private package access.', + inputSchema: { + token: z.string().describe('NPM auth token'), + registry_url: z.string().optional().describe('Custom NPM registry URL (e.g. "https://registry.npmjs.org/" or private Verdaccio/Artifactory URL)'), + scope: z.string().optional().describe('Package scope to associate this auth token with (e.g. "@myorg")'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.saveNpmAuth(params); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to save npm auth configuration', error); + } + } + ); + + // 8. delete_npm_auth + server.registerTool( + 'delete_npm_auth', + { + description: 'Delete saved npm authentication credentials for a specific scope or registry.', + inputSchema: { + target: z.string().optional().describe('Scope or ID identifier to remove, or omit to clear configuration'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.deleteNpmAuth(params.target); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to delete npm auth configuration', error); + } + } + ); +} diff --git a/packages/mcp-server/src/tools/providers.ts b/packages/mcp-server/src/tools/providers.ts new file mode 100644 index 0000000..b03ab99 --- /dev/null +++ b/packages/mcp-server/src/tools/providers.ts @@ -0,0 +1,208 @@ +/** + * Tools: Provider Management & Rate Limiting + */ + +import { z } from 'zod'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { UserCredentials } from '../auth.js'; +import { ApiClient } from '../api-client.js'; +import { formatApiResponse, formatToolError } from './response.js'; + +export function registerProviderTools(server: McpServer, getCredentials: () => UserCredentials) { + // 1. list_providers + server.registerTool( + 'list_providers', + { + description: 'List all configured provider instances across all notification channels, including their plugin package name, enabled status, and provider options.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.listProviders(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to list providers', error); + } + } + ); + + // 2. get_provider + server.registerTool( + 'get_provider', + { + description: 'Retrieve detailed configuration and status for a single provider instance by its ID. Credentials are masked by default.', + inputSchema: { + id: z.string().describe('Unique identifier of the provider instance (e.g. "sendgrid-primary", "twilio-marketing")'), + include_decrypted: z.boolean().optional().describe('Whether to reveal decrypted credentials instead of masked values (default: false)'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getProvider(params.id, params.include_decrypted); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to get provider '${params.id}'`, error); + } + } + ); + + // 3. create_provider + server.registerTool( + 'create_provider', + { + description: 'Instantiate and register a new provider with required credentials, options (e.g. rate limits), and enable state.', + inputSchema: { + id: z.string().describe('Unique user-defined identifier for this provider instance (e.g. "sendgrid-backup", "resend-prod")'), + plugin_name: z.string().describe('The installed plugin package name that powers this provider (e.g. "@simplens/smtp", "@simplens/sendgrid")'), + credentials: z.record(z.string(), z.unknown()).describe('Credential object matching the plugin schema (e.g. { apiKey: "..." } or { host: "...", port: 587 })'), + options: z.record(z.string(), z.unknown()).optional().describe('Optional provider parameters such as rate limits { rateLimit: { windowMs: 60000, maxRequests: 100 } }'), + enabled: z.boolean().optional().describe('Whether the provider is immediately active and available for routing (default: true)'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.createProvider(params); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to create provider '${params.id}'`, error); + } + } + ); + + // 4. update_provider + server.registerTool( + 'update_provider', + { + description: 'Update credentials, options, or enabled status of an existing provider instance.', + inputSchema: { + id: z.string().describe('Provider instance ID to update'), + credentials: z.record(z.string(), z.unknown()).optional().describe('New credentials object (only provide fields you want to update or overwrite)'), + options: z.record(z.string(), z.unknown()).optional().describe('Updated provider options or rate limits'), + enabled: z.boolean().optional().describe('Toggle provider enabled status'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const { id, ...payload } = params; + const result = await client.updateProvider(id, payload); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to update provider '${params.id}'`, error); + } + } + ); + + // 5. delete_provider + server.registerTool( + 'delete_provider', + { + description: 'Delete a provider instance. Fails if the provider is currently referenced as a primary or fallback provider in channel routing.', + inputSchema: { + id: z.string().describe('Provider ID to delete'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.deleteProvider(params.id); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to delete provider '${params.id}'`, error); + } + } + ); + + // 6. test_provider + server.registerTool( + 'test_provider', + { + description: 'Test provider connection and credentials without saving to the database. Can test an existing provider by provider_id or test raw un-persisted credentials.', + inputSchema: { + provider_id: z.string().optional().describe('ID of an existing saved provider to test'), + plugin_name: z.string().optional().describe('Plugin package name to test (required if testing new raw credentials)'), + credentials: z.record(z.string(), z.unknown()).optional().describe('Credentials to test against the provider service'), + options: z.record(z.string(), z.unknown()).optional().describe('Optional parameters to test with'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.testProvider(params); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to test provider connection', error); + } + } + ); + + // 7. get_providers_rate_limits + server.registerTool( + 'get_providers_rate_limits', + { + description: 'Get real-time rate limiter status, token bucket counts, and telemetry across all configured providers in Redis.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getProvidersRateLimits(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to retrieve providers rate limits', error); + } + } + ); + + // 8. get_provider_rate_limit + server.registerTool( + 'get_provider_rate_limit', + { + description: 'Get real-time rate limit status, remaining token capacity, and reset window for a specific provider instance.', + inputSchema: { + id: z.string().describe('Provider instance ID'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getProviderRateLimit(params.id); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to retrieve rate limit for provider '${params.id}'`, error); + } + } + ); + + // 9. reset_provider_rate_limit + server.registerTool( + 'reset_provider_rate_limit', + { + description: 'Manually reset rate limiter tokens and telemetry counters for a specific provider instance in Redis.', + inputSchema: { + id: z.string().describe('Provider instance ID to reset rate limiter for'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.resetProviderRateLimit(params.id); + return formatApiResponse(result); + } catch (error) { + return formatToolError(`Failed to reset rate limit for provider '${params.id}'`, error); + } + } + ); +} diff --git a/packages/mcp-server/src/tools/settings.ts b/packages/mcp-server/src/tools/settings.ts new file mode 100644 index 0000000..ae39422 --- /dev/null +++ b/packages/mcp-server/src/tools/settings.ts @@ -0,0 +1,70 @@ +/** + * Tools: Dynamic Operational Configuration Settings + */ + +import { z } from 'zod'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { UserCredentials } from '../auth.js'; +import { ApiClient } from '../api-client.js'; +import { formatApiResponse, formatToolError } from './response.js'; + +export function registerSettingsTools(server: McpServer, getCredentials: () => UserCredentials) { + // 1. get_operational_settings + server.registerTool( + 'get_operational_settings', + { + description: 'Retrieve current dynamic operational settings (worker concurrency, retry limits, delayed and recovery intervals, failure alert thresholds).', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.getOperationalSettings(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to get operational settings', error); + } + } + ); + + // 2. update_operational_settings + server.registerTool( + 'update_operational_settings', + { + description: 'Hot-update operational settings across the SimpleNS cluster without requiring service restarts. Changes are broadcast via Redis pub/sub.', + inputSchema: { + settings: z.record(z.string(), z.unknown()).describe('Partial operational settings object to merge and update (e.g. { notification_worker: { batch_size: 20 }, delayed_processor: { poll_interval_ms: 3000 } })'), + }, + }, + async (params) => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.updateOperationalSettings(params.settings); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to update operational settings', error); + } + } + ); + + // 3. reset_operational_settings + server.registerTool( + 'reset_operational_settings', + { + description: 'Reset all dynamic operational settings back to initial system defaults.', + inputSchema: {}, + }, + async () => { + try { + const credentials = getCredentials(); + const client = new ApiClient(credentials); + const result = await client.resetOperationalSettings(); + return formatApiResponse(result); + } catch (error) { + return formatToolError('Failed to reset operational settings', error); + } + } + ); +} diff --git a/packages/mcp-server/tests/verify-stdio.ts b/packages/mcp-server/tests/verify-stdio.ts index ea213f2..2f4c250 100644 --- a/packages/mcp-server/tests/verify-stdio.ts +++ b/packages/mcp-server/tests/verify-stdio.ts @@ -33,8 +33,8 @@ async function main() { console.log(`Found ${tools.tools.length} tools:`); tools.tools.forEach(t => console.log(`- ${t.name}`)); - if (tools.tools.length !== 29) { - throw new Error(`Expected 29 tools, found ${tools.tools.length}`); + if (tools.tools.length !== 54) { + throw new Error(`Expected 54 tools, found ${tools.tools.length}`); } diff --git a/packages/onboard/README.md b/packages/onboard/README.md index d23ba98..bde3bed 100644 --- a/packages/onboard/README.md +++ b/packages/onboard/README.md @@ -79,22 +79,22 @@ npx @simplens/onboard --infra --dir /path/to/setup ```bash # Complete setup with all options via CLI (no prompts) -npx @simplens/onboard --full --infra mongo kafka redis nginx --env default --base-path /dashboard --plugin @simplens/mock @simplens/nodemailer-gmail --no-output +npx @simplens/onboard --full --infra mongo kafka redis nginx --env default --base-path /dashboard --app-version latest --skip-plugins --no-output ``` This mode: - Requires `--full` flag to enable non-interactive mode - Requires `--env ` to specify environment mode -- **Auto-generates secure credentials** for NS_API_KEY, AUTH_SECRET, and ADMIN_PASSWORD -- **Auto-generates placeholder credentials** for plugins -- **Auto-generates version values** for CORE_VERSION and DASHBOARD_VERSION +- **Auto-generates secure credentials** for `JWT_SECRET` (and `AUTH_SECRET`/`NS_API_KEY`/`ADMIN_PASSWORD` for legacy SimpleNS <= 1.3.0) +- **Auto-generates placeholder credentials** for plugins if `--plugin` is provided +- **Auto-generates version value** for `VERSION` (default: `latest`) - All other options are optional with sensible defaults - Services are not auto-started (use `docker compose up -d` manually; fallback: `docker-compose up -d`) **⚠️ IMPORTANT**: Auto-generated credentials are **NOT secure for production**. After setup completes, you **must** update the following in your `.env` file: -- `NS_API_KEY` - API authentication key -- `AUTH_SECRET` - Session secret for dashboard -- `ADMIN_PASSWORD` - Dashboard admin password +- `JWT_SECRET` - Secret key for API authentication tokens (for legacy <= 1.3.0, `AUTH_SECRET` was used for dashboard session) +- `NS_API_KEY` - API authentication key (legacy SimpleNS <= 1.3.0 only; for > 1.3.0, external API keys are created and managed dynamically in the Dashboard at `/keys`) +- `ADMIN_PASSWORD` - Dashboard admin password (legacy SimpleNS <= 1.3.0 only; for > 1.3.0, the admin account is set up interactively via the Dashboard at `/setup` upon first boot) - Plugin credentials (if any plugins were installed) The CLI will display a security notice with all credentials that need to be updated. @@ -108,9 +108,11 @@ The CLI will display a security notice with all credentials that need to be upda | `--env ` | Environment setup mode: `default` or `interactive` | Prompted | | `--dir ` | Target directory for setup files | Current directory | | `--base-path ` | Dashboard base path (example: `/dashboard`) | Empty (root) | -| `--core-version ` | Override `CORE_VERSION` in generated `.env` (primarily for `--full`) | `latest` | -| `--dashboard-version ` | Override `DASHBOARD_VERSION` in generated `.env` (primarily for `--full`) | `latest` | +| `--app-version ` | Override `VERSION` in generated `.env` and compose (alias: `--version-tag`) | `latest` | +| `--core-version ` | Backward-compatible alias for `--app-version` | `latest` | +| `--dashboard-version ` | Backward-compatible alias for `--app-version` | `latest` | | `--plugin [plugins...]` | Plugins to install (e.g., `@simplens/mock @simplens/nodemailer-gmail`) | Prompted | +| `--skip-plugins` | Skip plugin configuration during onboarding (install via Dashboard later) | `false` | | `--ssl` | Enable optional SSL automation with Certbot | `false` | | `--ssl-domain ` | Public domain for SSL cert (required with `--ssl` in `--full`) | Prompted | | `--ssl-email ` | Email for Let's Encrypt registration (required with `--ssl` in `--full`) | Prompted | @@ -126,6 +128,19 @@ The CLI will display a security notice with all credentials that need to be upda - `loki` - Loki log aggregation (optional) - `grafana` - Grafana observability dashboard (optional) +## SimpleNS Version Compatibility (> 1.3.0 vs <= 1.3.0) + +SimpleNS versions **> 1.3.0** feature a modernized plugin management system and database-driven dynamic configuration. `@simplens/onboard` automatically selects the appropriate compose templates, volume mounts, and `.env` format based on the selected version: + +- **Unified Image Tag (`VERSION`)**: Both core services (`api`, `worker`, `processors`, `recovery`) and `dashboard` share a single `VERSION` variable in `.env` and `docker-compose.yaml` (default: `latest`). +- **SimpleNS > 1.3.0**: + - **No Shared Plugins Volume**: Services have their own internal plugin directory. The shared `plugin-data` volume is completely removed. + - **Optional Plugins Configuration**: Configuring plugins via `simplens.config.yaml` during onboarding is completely optional. You can skip plugin setup during onboarding (`--skip-plugins`) and install plugins dynamically from the Admin Dashboard. If you configure plugins during onboarding, `simplens.config.yaml` is mounted and `SIMPLENS_CONFIG_PATH` is passed to the container. + - **First-Boot Dashboard Admin Setup**: `ADMIN_USERNAME` and `ADMIN_PASSWORD` are not stored in `.env`. On first launch, navigate to the Dashboard to initialize the administrator account via the guided setup screen (`/setup`). + - **Dynamic Operational Settings**: Operational configurations (outbox interval/batch size, idempotency TTL, recovery timeouts, cleanup retention) are stored in MongoDB and dynamically updated live via the Dashboard Settings page. +- **SimpleNS <= 1.3.0**: + - Automatically activates legacy mode with the shared `plugin-data` volume, required `simplens.config.yaml`, `ADMIN_PASSWORD` in `.env`, and operational settings in `.env`. + ## Workflow 1. **Prerequisites Check** @@ -144,15 +159,12 @@ The CLI will display a security notice with all credentials that need to be upda - Load defaults from `.env.example` - Auto-fill infra connection URLs - Ask for `BASE_PATH` first and reuse it throughout setup - - Prompt for critical values (API keys, passwords) + - Prompt for critical values (API keys, session secret) - Generate `.env` file 4. **Plugin Installation** - - Fetch official plugins from registry - - Interactive multi-select - - Generate `simplens.config.yaml` - - Extract and prompt for plugin credentials - - Append credentials to `.env` + - For SimpleNS > 1.3.0: Plugins are optional. Choose to configure plugins now or skip (`--skip-plugins`) and install later via Dashboard. + - For SimpleNS <= 1.3.0: Interactive plugin selection, generate `simplens.config.yaml`, and append credentials to `.env`. 5. **Service Orchestration** - Optionally start infrastructure services @@ -252,8 +264,7 @@ npx @simplens/onboard \ --full \ --infra mongo kafka redis nginx \ --env default \ - --core-version 1.2 \ - --dashboard-version 1.2 \ + --app-version latest \ --base-path /dashboard \ --plugin @simplens/mock @simplens/nodemailer-gmail \ --dir ./my-simplens-setup \ @@ -271,8 +282,8 @@ npx @simplens/onboard \ npx @simplens/onboard \ --full \ --env default \ - --core-version 1.2 \ - --dashboard-version 1.2 \ + --app-version latest \ + --skip-plugins \ --dir /app/simplens # Then start services in CI: diff --git a/packages/onboard/package-lock.json b/packages/onboard/package-lock.json index dac26a2..34dd339 100644 --- a/packages/onboard/package-lock.json +++ b/packages/onboard/package-lock.json @@ -1,12 +1,12 @@ { "name": "@simplens/onboard", - "version": "1.0.12", + "version": "1.0.13", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@simplens/onboard", - "version": "1.0.12", + "version": "1.0.13", "license": "ISC", "dependencies": { "@clack/prompts": "^1.0.0", diff --git a/packages/onboard/package.json b/packages/onboard/package.json index ed61298..cd7e382 100644 --- a/packages/onboard/package.json +++ b/packages/onboard/package.json @@ -1,6 +1,6 @@ { "name": "@simplens/onboard", - "version": "1.0.12", + "version": "1.0.13", "type": "module", "main": "dist/index.js", "engines": { diff --git a/packages/onboard/src/__tests__/env-config.test.ts b/packages/onboard/src/__tests__/env-config.test.ts index 3d67beb..de376a2 100644 --- a/packages/onboard/src/__tests__/env-config.test.ts +++ b/packages/onboard/src/__tests__/env-config.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { normalizeBasePath, validateBasePath } from '../env-config.js'; +import { normalizeBasePath, validateBasePath, isVersionGreaterThan130, loadEnvExample } from '../env-config.js'; describe('env-config base path helpers', () => { it('accepts empty base path for root', () => { @@ -26,3 +26,69 @@ describe('env-config base path helpers', () => { expect(normalizeBasePath(' /dashboard ')).toBe('/dashboard'); }); }); + +describe('isVersionGreaterThan130 version comparison', () => { + it('treats latest, master, and empty as > 1.3.0', () => { + expect(isVersionGreaterThan130('latest')).toBe(true); + expect(isVersionGreaterThan130('master')).toBe(true); + expect(isVersionGreaterThan130(undefined)).toBe(true); + expect(isVersionGreaterThan130('')).toBe(true); + }); + + it('identifies versions <= 1.3.0 as legacy', () => { + expect(isVersionGreaterThan130('1.3.0')).toBe(false); + expect(isVersionGreaterThan130('v1.3.0')).toBe(false); + expect(isVersionGreaterThan130('1.2.9')).toBe(false); + expect(isVersionGreaterThan130('1.0.0')).toBe(false); + expect(isVersionGreaterThan130('0.9.0')).toBe(false); + }); + + it('identifies versions > 1.3.0 as new', () => { + expect(isVersionGreaterThan130('1.3.1')).toBe(true); + expect(isVersionGreaterThan130('v1.3.1')).toBe(true); + expect(isVersionGreaterThan130('1.4.0')).toBe(true); + expect(isVersionGreaterThan130('2.0.0')).toBe(true); + }); +}); + +describe('loadEnvExample templates', () => { + it('returns essential variables matching ./env.example for new versions (> 1.3.0)', async () => { + const envVars = await loadEnvExample(false); + const keys = envVars.map(v => v.key); + + expect(keys).not.toContain('NS_API_KEY'); + expect(keys).toContain('MONGO_URI'); + expect(keys).toContain('BROKERS'); + expect(keys).toContain('REDIS_URL'); + expect(keys).not.toContain('AUTH_SECRET'); + expect(keys).toContain('JWT_SECRET'); + expect(keys).toContain('VERSION'); + expect(keys).not.toContain('CORE_VERSION'); + expect(keys).not.toContain('DASHBOARD_VERSION'); + + // Must NOT contain admin credentials or operational settings + expect(keys).not.toContain('ADMIN_USERNAME'); + expect(keys).not.toContain('ADMIN_PASSWORD'); + expect(keys).not.toContain('OUTBOX_POLL_INTERVAL_MS'); + expect(keys).not.toContain('OUTBOX_BATCH_SIZE'); + expect(keys).not.toContain('RECOVERY_BATCH_SIZE'); + expect(keys).not.toContain('IDEMPOTENCY_TTL_SECONDS'); + expect(keys).not.toContain('HTTPS_COOKIE'); + }); + + it('returns legacy variables including admin credentials and outbox settings for legacy (<= 1.3.0)', async () => { + const envVars = await loadEnvExample(true); + const keys = envVars.map(v => v.key); + + expect(keys).toContain('NS_API_KEY'); + expect(keys).toContain('VERSION'); + expect(keys).not.toContain('CORE_VERSION'); + expect(keys).not.toContain('DASHBOARD_VERSION'); + expect(keys).toContain('ADMIN_USERNAME'); + expect(keys).toContain('ADMIN_PASSWORD'); + expect(keys).toContain('OUTBOX_POLL_INTERVAL_MS'); + expect(keys).toContain('OUTBOX_BATCH_SIZE'); + expect(keys).toContain('RECOVERY_BATCH_SIZE'); + expect(keys).toContain('SIMPLENS_CONFIG_PATH'); + }); +}); diff --git a/packages/onboard/src/__tests__/infra.test.ts b/packages/onboard/src/__tests__/infra.test.ts index 74015f7..8e2d8dd 100644 --- a/packages/onboard/src/__tests__/infra.test.ts +++ b/packages/onboard/src/__tests__/infra.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { buildAppComposeContent } from '../infra.js'; +import { buildAppComposeContent, generateNginxConfig } from '../infra.js'; +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; describe('infra app compose generation', () => { it('does not include nginx when disabled', () => { @@ -27,4 +30,84 @@ describe('infra app compose generation', () => { expect(compose).toContain('certbot-etc:'); expect(compose).toContain('certbot-www:'); }); + + it('generates legacy compose with shared plugin-data volume and static config mount when isLegacy is true', () => { + const compose = buildAppComposeContent(false, { isLegacy: true }); + expect(compose).toContain('plugin-data:/app/.plugins'); + expect(compose).toContain('plugin-data:'); + expect(compose).toContain('./simplens.config.yaml:/app/simplens.config.yaml:ro'); + expect(compose).toContain('SIMPLENS_CONFIG_PATH'); + }); + + it('generates new compose (> 1.3.0) without plugin-data volume and without config mount when plugins skipped', () => { + const compose = buildAppComposeContent(false, { isLegacy: false, hasPluginsConfig: false }); + expect(compose).not.toContain('plugin-data'); + expect(compose).not.toContain('simplens.config.yaml'); + expect(compose).not.toContain('SIMPLENS_CONFIG_PATH'); + expect(compose).toContain('logs-data:'); + }); + + it('generates new compose (> 1.3.0) with config mount and SIMPLENS_CONFIG_PATH when plugins are configured', () => { + const compose = buildAppComposeContent(false, { isLegacy: false, hasPluginsConfig: true }); + expect(compose).not.toContain('plugin-data'); + expect(compose).toContain('./simplens.config.yaml:/app/simplens.config.yaml:ro'); + expect(compose).toContain('SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml}'); + expect(compose).toContain('logs-data:'); + }); +}); + +describe('generateNginxConfig', () => { + it('generates nginx config for root basePath correctly', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'onboard-test-')); + try { + await generateNginxConfig(tmpDir, ''); + const content = await fs.readFile(path.join(tmpDir, 'nginx.conf'), 'utf-8'); + expect(content).toContain('location ^~ /api {'); + expect(content).toContain('proxy_pass http://api:3000;'); + expect(content).toContain('location / {'); + expect(content).toContain('proxy_pass http://dashboard:3002;'); + expect(content).not.toContain('return 302'); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); + + it('generates nginx config for legacy version (<= 1.3.0) with custom basePath correctly', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'onboard-test-')); + try { + await generateNginxConfig(tmpDir, '/sns', { version: '1.3.0' }); + const content = await fs.readFile(path.join(tmpDir, 'nginx.conf'), 'utf-8'); + expect(content).toContain('location /api {'); + expect(content).not.toContain('location ^~ /sns/api {'); + expect(content).not.toContain('location ^~ /api {'); + expect(content).not.toContain('return 302'); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); + + it('generates nginx config for legacy version using isLegacy flag', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'onboard-test-')); + try { + await generateNginxConfig(tmpDir, '/sns', { isLegacy: true }); + const content = await fs.readFile(path.join(tmpDir, 'nginx.conf'), 'utf-8'); + expect(content).toContain('location /api {'); + expect(content).not.toContain('return 302'); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); + + it('generates nginx config for version > 1.3.0 with custom basePath correctly', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'onboard-test-')); + try { + await generateNginxConfig(tmpDir, '/sns', { version: '1.4.0' }); + const content = await fs.readFile(path.join(tmpDir, 'nginx.conf'), 'utf-8'); + expect(content).toContain('location ^~ /sns/api {'); + expect(content).toContain('return 302 /sns;'); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); }); + diff --git a/packages/onboard/src/__tests__/validators.test.ts b/packages/onboard/src/__tests__/validators.test.ts index ec8a603..cd32cb5 100644 --- a/packages/onboard/src/__tests__/validators.test.ts +++ b/packages/onboard/src/__tests__/validators.test.ts @@ -177,6 +177,7 @@ describe('validators', () => { it('should reject short secrets', () => { expect(validateEnvValue('AUTH_SECRET', 'abc')).toBe(false); + expect(validateEnvValue('JWT_SECRET', 'abc')).toBe(false); }); it('should accept API keys with 8+ characters', () => { diff --git a/packages/onboard/src/config/constants.ts b/packages/onboard/src/config/constants.ts index fd808ca..22e364e 100644 --- a/packages/onboard/src/config/constants.ts +++ b/packages/onboard/src/config/constants.ts @@ -35,17 +35,27 @@ export const DOCKER_TIMEOUTS = { } as const; /** - * Critical environment variables that always need user input + * Legacy critical environment variables (SimpleNS <= 1.3.0) */ -export const CRITICAL_ENV_KEYS = [ +export const LEGACY_CRITICAL_ENV_KEYS = [ 'NS_API_KEY', 'MONGO_URI', 'BROKERS', 'REDIS_URL', 'AUTH_SECRET', 'ADMIN_PASSWORD', - 'CORE_VERSION', - 'DASHBOARD_VERSION', + 'VERSION', +]; + +/** + * Critical environment variables that always need user input (SimpleNS > 1.3.0) + */ +export const CRITICAL_ENV_KEYS = [ + 'MONGO_URI', + 'BROKERS', + 'REDIS_URL', + 'JWT_SECRET', + 'VERSION', ]; /** diff --git a/packages/onboard/src/env-config.ts b/packages/onboard/src/env-config.ts index 7533e26..c281294 100644 --- a/packages/onboard/src/env-config.ts +++ b/packages/onboard/src/env-config.ts @@ -3,11 +3,36 @@ import { text, password } from '@clack/prompts'; import { handleCancel } from './ui.js'; import path from 'path'; import crypto from 'crypto'; -import { CRITICAL_ENV_KEYS } from './config/constants.js'; +import { CRITICAL_ENV_KEYS, LEGACY_CRITICAL_ENV_KEYS } from './config/constants.js'; import type { EnvVariable } from './types/domain.js'; export const DEFAULT_BASE_PATH = ''; -const AUTO_GENERATE_ON_EMPTY_KEYS = ['NS_API_KEY', 'AUTH_SECRET', 'CORE_VERSION', 'DASHBOARD_VERSION'] as const; + +/** + * Check if the provided SimpleNS version is greater than 1.3.0. + * Versions <= 1.3.0 are considered legacy. + * 'latest', 'master', empty/undefined, or non-numeric tags default to true (> 1.3.0). + */ +export function isVersionGreaterThan130(version: string | undefined): boolean { + if (!version || version === 'latest' || version === 'master') { + return true; + } + const clean = version.trim().replace(/^v/, ''); + const match = clean.match(/^(\d+)\.(\d+)(?:\.(\d+))?/); + if (!match) { + return true; + } + const major = parseInt(match[1], 10); + const minor = parseInt(match[2], 10); + const patch = match[3] !== undefined ? parseInt(match[3], 10) : 0; + + if (major > 1) return true; + if (major < 1) return false; + if (minor > 3) return true; + if (minor < 3) return false; + return patch > 0; +} +const AUTO_GENERATE_ON_EMPTY_KEYS = ['NS_API_KEY', 'AUTH_SECRET', 'JWT_SECRET', 'VERSION'] as const; function shouldAutoGenerateOnEmpty(key: string, fullMode: boolean): boolean { return !fullMode && AUTO_GENERATE_ON_EMPTY_KEYS.includes(key as (typeof AUTO_GENERATE_ON_EMPTY_KEYS)[number]); @@ -35,13 +60,13 @@ export function generateDefaultValue(key: string): string { if (key === 'NS_API_KEY') { return `ns_${generateSecureRandom(48)}`; } - if (key === 'AUTH_SECRET') { + if (key === 'AUTH_SECRET' || key === 'JWT_SECRET') { return generateSecureRandom(64); } if (key === 'ADMIN_PASSWORD') { return `Admin${generateSecureRandom(16)}`; } - if (key === 'CORE_VERSION' || key ==='DASHBOARD_VERSION') { + if (key === 'VERSION') { return `latest`; } return ''; @@ -87,7 +112,7 @@ export function normalizeBasePath(input: string): string { */ export async function promptBasePath(defaultValue: string = DEFAULT_BASE_PATH): Promise { const result = await text({ - message: 'BASE_PATH for dashboard (leave empty for root, example: /dashboard):', + message: 'BASE_PATH for API and dashboard (leave empty for root, example: /sns):', placeholder: defaultValue || 'leave empty for root', defaultValue, validate: (value: string | undefined) => { @@ -101,12 +126,50 @@ export async function promptBasePath(defaultValue: string = DEFAULT_BASE_PATH): return normalizeBasePath(result as string); } -/** - * Load and parse .env.example from embedded template - */ -export async function loadEnvExample(): Promise { - // Embedded .env template - always available regardless of installation - const envTemplate = ` +export const ESSENTIAL_ENV_TEMPLATE = ` +NODE_ENV=production +# ============================================ +# API SERVER +# ============================================ +PORT=3000 +JWT_SECRET= + +# ============================================ +# DATABASE +# ============================================ +MONGO_URI= + +# ============================================ +# KAFKA +# ============================================ +BROKERS= + +# ============================================ +# REDIS +# ============================================ +REDIS_URL= + +# ============================================ +# LOGGING (Grafana/Loki) +# ============================================ +LOKI_URL= +LOG_LEVEL=info +LOG_TO_FILE=true + +# ============================================ +# DOCKER IMAGE VERSION +# ============================================ +VERSION=latest + +# ============================================ +# ADMIN DASHBOARD +# ============================================ +API_BASE_URL=http://api:3000 +DASHBOARD_PORT=3002 +BASE_PATH= +`; + +export const LEGACY_ENV_TEMPLATE = ` NODE_ENV=production # ============================================ # API SERVER @@ -186,10 +249,9 @@ LOG_LEVEL=info LOG_TO_FILE=true # ============================================ -# DOCKER IMAGE VERSIONS +# DOCKER IMAGE VERSION # ============================================ -CORE_VERSION=latest -DASHBOARD_VERSION=latest +VERSION=latest # ============================================ # ADMIN DASHBOARD @@ -205,13 +267,10 @@ BASE_PATH= DASHBOARD_PORT=3002 `; - return parseEnvContent(envTemplate); -} - /** * Parse .env content into structured format */ -function parseEnvContent(content: string): EnvVariable[] { +function parseEnvContent(content: string, criticalKeys: readonly string[] = CRITICAL_ENV_KEYS): EnvVariable[] { const lines = content.split('\n'); const variables: EnvVariable[] = []; let currentComment = ''; @@ -237,9 +296,9 @@ function parseEnvContent(content: string): EnvVariable[] { const [, key, value] = match; variables.push({ key, - value: value || '', + value: value ? value.trim() : '', description: currentComment || undefined, - required: CRITICAL_ENV_KEYS.includes(key) || !value, + required: criticalKeys.includes(key) || !value, }); currentComment = ''; } @@ -248,6 +307,15 @@ function parseEnvContent(content: string): EnvVariable[] { return variables; } +/** + * Load and parse .env.example from embedded template + */ +export async function loadEnvExample(isLegacy: boolean = false): Promise { + const template = isLegacy ? LEGACY_ENV_TEMPLATE : ESSENTIAL_ENV_TEMPLATE; + const criticalKeys = isLegacy ? LEGACY_CRITICAL_ENV_KEYS : CRITICAL_ENV_KEYS; + return parseEnvContent(template, criticalKeys); +} + /** * Prompts user for environment variable values based on the selected mode. * @@ -262,11 +330,15 @@ export async function promptEnvVariables( infraServices: string[], basePath: string = DEFAULT_BASE_PATH, fullMode: boolean = false, - envOverrides: Partial> = {} + envOverrides: Partial> = {}, + version?: string ): Promise> { logInfo('Configuring environment variables...'); - const envVars = await loadEnvExample(); + const effectiveVersion = envOverrides.VERSION || envOverrides.CORE_VERSION || envOverrides.DASHBOARD_VERSION || version || 'latest'; + const isLegacy = !isVersionGreaterThan130(effectiveVersion); + const criticalKeys = isLegacy ? LEGACY_CRITICAL_ENV_KEYS : CRITICAL_ENV_KEYS; + const envVars = await loadEnvExample(isLegacy); const result = new Map(); const normalizedBasePath = normalizeBasePath(basePath); const basePathLabel = normalizedBasePath || '(root)'; @@ -291,8 +363,8 @@ export async function promptEnvVariables( if (mode === 'default') { // Use defaults, only prompt for critical values for (const envVar of envVars) { - if (fullMode && envVar.key in envOverrides) { - const overrideValue = envOverrides[envVar.key as keyof typeof envOverrides]; + if (fullMode && (envVar.key in envOverrides || (envVar.key === 'VERSION' && ('CORE_VERSION' in envOverrides || 'VERSION' in envOverrides)))) { + const overrideValue = envOverrides[envVar.key as keyof typeof envOverrides] || (envVar.key === 'VERSION' ? (envOverrides.VERSION || envOverrides.CORE_VERSION) : undefined); if (overrideValue) { result.set(envVar.key, overrideValue); continue; @@ -311,14 +383,20 @@ export async function promptEnvVariables( continue; } + // VERSION collected upfront or overridden + if (envVar.key === 'VERSION') { + result.set(envVar.key, effectiveVersion); + continue; + } + // Use default value if available - if (envVar.value && !CRITICAL_ENV_KEYS.includes(envVar.key)) { + if (envVar.value && !criticalKeys.includes(envVar.key)) { result.set(envVar.key, envVar.value); continue; } // Prompt for critical values (only if not auto-filled) - if (CRITICAL_ENV_KEYS.includes(envVar.key)) { + if (criticalKeys.includes(envVar.key)) { if (fullMode) { // In full mode, auto-generate critical values const defaultValue = generateDefaultValue(envVar.key); @@ -327,7 +405,7 @@ export async function promptEnvVariables( } } else { // Show changelog info for version variables - if (envVar.key === 'CORE_VERSION' || envVar.key === 'DASHBOARD_VERSION') { + if (envVar.key === 'VERSION') { logInfo('ℹ️ Visit https://simplens.in/changelog for version information'); } @@ -375,8 +453,8 @@ export async function promptEnvVariables( logInfo('Interactive mode: You will be prompted for each environment variable.'); for (const envVar of envVars) { - if (fullMode && envVar.key in envOverrides) { - const overrideValue = envOverrides[envVar.key as keyof typeof envOverrides]; + if (fullMode && (envVar.key in envOverrides || (envVar.key === 'VERSION' && ('CORE_VERSION' in envOverrides || 'VERSION' in envOverrides)))) { + const overrideValue = envOverrides[envVar.key as keyof typeof envOverrides] || (envVar.key === 'VERSION' ? (envOverrides.VERSION || envOverrides.CORE_VERSION) : undefined); if (overrideValue) { result.set(envVar.key, overrideValue); continue; @@ -390,9 +468,15 @@ export async function promptEnvVariables( result.set(envVar.key, normalizedBasePath); continue; } + + // VERSION collected upfront or overridden + if (envVar.key === 'VERSION') { + result.set(envVar.key, effectiveVersion); + continue; + } // Show changelog info for version variables - if (envVar.key === 'CORE_VERSION' || envVar.key === 'DASHBOARD_VERSION') { + if (envVar.key === 'VERSION') { logInfo('ℹ️ Visit https://simplens.in/changelog for version information'); } @@ -443,7 +527,7 @@ export async function promptEnvVariables( * Get suggested value for specific keys */ function getSuggestedValue(key: string): string { - if (key === 'NS_API_KEY' || key === 'AUTH_SECRET') { + if (key === 'NS_API_KEY' || key === 'AUTH_SECRET' || key === 'JWT_SECRET') { return `Replace with: openssl rand -base64 32 (or) Press to auto-generate value`; } if (key === 'NODE_ENV') { @@ -452,7 +536,7 @@ function getSuggestedValue(key: string): string { if (key === 'ADMIN_USERNAME') { return 'admin'; } - if (key === 'CORE_VERSION' || key === 'DASHBOARD_VERSION') { + if (key === 'VERSION' || key === 'CORE_VERSION' || key === 'DASHBOARD_VERSION') { return 'latest'; } return ''; @@ -463,12 +547,23 @@ function getSuggestedValue(key: string): string { */ export async function generateEnvFile( targetDir: string, - envVars: Map + envVars: Map, + isLegacy: boolean = false ): Promise { const envPath = path.join(targetDir, '.env'); let content = '# SimpleNS Environment Configuration\n'; - content += '# Generated by @simplens/onboard\n\n'; + content += '# Generated by @simplens/onboard\n'; + if (!isLegacy) { + content += '#\n'; + content += '# NOTE: All operational settings (worker batch sizes, poll intervals, retries,\n'; + content += '# deduplication TTLs, recovery thresholds, and log levels) are now managed\n'; + content += '# dynamically via MongoDB and the Admin Dashboard with live zero-downtime hot-reloading!\n'; + content += '#\n'; + content += '# Administrator credentials are configured interactively on first launch via\n'; + content += '# the Dashboard Setup UI (/setup) and stored securely in MongoDB.\n'; + } + content += '\n'; for (const [key, value] of envVars.entries()) { content += `${key}=${value}\n`; diff --git a/packages/onboard/src/index.ts b/packages/onboard/src/index.ts index 2a02efc..592569c 100644 --- a/packages/onboard/src/index.ts +++ b/packages/onboard/src/index.ts @@ -36,6 +36,7 @@ import { normalizeBasePath, validateBasePath, DEFAULT_BASE_PATH, + isVersionGreaterThan130, } from './env-config.js'; import { fetchAvailablePlugins, @@ -62,15 +63,18 @@ const program = new Command(); program .name('@simplens/onboard') .description('A CLI tool to setup a SimpleNS instance on your machine/server') - .version('1.0.0') + .version('1.0.13') .option('--full', 'Non-interactive mode - all options must be provided via CLI') .option('--infra [services...]', 'Infrastructure services (mongo, kafka, kafka-ui, redis, nginx, loki, grafana)') .option('--env ', 'Environment setup mode: "default" or "interactive"') .option('--dir ', 'Target directory for setup') .option('--base-path ', 'Dashboard BASE_PATH (example: /dashboard, default: root)') - .option('--core-version ', 'Override CORE_VERSION in generated .env (primarily for --full mode)') - .option('--dashboard-version ', 'Override DASHBOARD_VERSION in generated .env (primarily for --full mode)') + .option('--app-version ', 'Override VERSION in generated .env (primarily for --full mode)') + .option('--version-tag ', 'Override VERSION in generated .env (alias)') + .option('--core-version ', 'Alias for --app-version / VERSION') + .option('--dashboard-version ', 'Alias for --app-version / VERSION') .option('--plugin [plugins...]', 'Plugins to install (e.g., @simplens/mock @simplens/nodemailer-gmail)') + .option('--skip-plugins', 'Skip plugin configuration (plugins can be installed via dashboard later)') .option('--ssl', 'Enable optional SSL certificate setup using Dockerized Certbot') .option('--ssl-domain ', 'Public domain for SSL certificate (required with --ssl in --full mode)') .option('--ssl-email ', 'Email for Let\'s Encrypt registration (required with --ssl in --full mode)') @@ -86,6 +90,8 @@ interface OnboardSetupOptions { enableSsl: boolean; sslDomain?: string; sslEmail?: string; + version: string; + skipPlugins?: boolean; } function printStep(step: number, total: number, title: string): void { @@ -134,6 +140,7 @@ function showSetupSummary(setupOptions: OnboardSetupOptions, targetDir: string, const summaryLines = [ `Target directory : ${targetDir}`, + `SimpleNS Version : ${setupOptions.version}`, `Infrastructure : ${infraLabel}`, `Environment mode : ${setupOptions.envMode}`, `BASE_PATH : ${basePathLabel}`, @@ -373,6 +380,27 @@ async function promptSetupOptions(options: any): Promise { } } + // --- Version --- + let versionValue: string; + const versionOption = (typeof options.appVersion === 'string' && options.appVersion.trim()) || + (typeof options.versionTag === 'string' && options.versionTag.trim()) || + (typeof options.coreVersion === 'string' && options.coreVersion.trim()) || + (typeof options.dashboardVersion === 'string' && options.dashboardVersion.trim()); + if (versionOption) { + versionValue = versionOption; + } else if (isFullMode) { + versionValue = 'latest'; + } else { + const result = await text({ + message: 'SimpleNS version to deploy (e.g. latest, 1.4.0, 1.3.0):', + defaultValue: 'latest', + placeholder: 'latest', + withGuide: true, + }); + handleCancel(result); + versionValue = (result as string).trim() || 'latest'; + } + return { infra: infraValue, infraServices: infraServices, @@ -383,6 +411,8 @@ async function promptSetupOptions(options: any): Promise { enableSsl: enableSslValue, sslDomain: sslDomainValue, sslEmail: sslEmailValue, + version: versionValue, + skipPlugins: options.skipPlugins === true, }; } @@ -416,6 +446,7 @@ async function main() { // Prompt for setup options if not provided const setupOptions = await promptSetupOptions(options); + const isLegacy = !isVersionGreaterThan130(setupOptions.version); // Get target directory const targetDir = path.resolve(setupOptions.targetDir); @@ -475,6 +506,8 @@ async function main() { await writeAppCompose(targetDir, { includeNginx: false, includeSsl: false, + isLegacy, + hasPluginsConfig: isLegacy ? true : (setupOptions.plugins.length > 0), }); const controlScriptFilename = await generateControlScript(targetDir, { os: detectOS(), @@ -484,10 +517,10 @@ async function main() { // Step 4: Environment configuration log.step('Step 4/6 — Environment Configuration'); const envMode = setupOptions.envMode; + const versionOverride = options.appVersion || options.versionTag || options.coreVersion || options.dashboardVersion; const envOverrides = options.full ? { - CORE_VERSION: options.coreVersion, - DASHBOARD_VERSION: options.dashboardVersion, + VERSION: versionOverride || setupOptions.version, } : undefined; const envVars = await promptEnvVariables( @@ -495,16 +528,24 @@ async function main() { selectedInfraServices, setupOptions.basePath, options.full || false, - envOverrides + envOverrides, + setupOptions.version ); - await generateEnvFile(targetDir, envVars); + await generateEnvFile(targetDir, envVars, isLegacy); // In full mode, notify user about auto-generated credentials if (options.full) { - logWarning( - '⚠️ Auto-generated credentials in .env file. ' + - 'Please update NS_API_KEY, AUTH_SECRET, and ADMIN_PASSWORD before deploying to production!' - ); + if (isLegacy) { + logWarning( + '⚠️ Auto-generated credentials in .env file. ' + + 'Please update NS_API_KEY, AUTH_SECRET, and ADMIN_PASSWORD before deploying to production!' + ); + } else { + logWarning( + '⚠️ Auto-generated credentials in .env file. ' + + 'Please update JWT_SECRET before deploying to production!' + ); + } } // Generate nginx.conf whenever nginx is active in either compose file @@ -514,6 +555,8 @@ async function main() { enableSsl: setupOptions.enableSsl, domain: setupOptions.sslDomain, sslMode: setupOptions.enableSsl ? 'bootstrap' : 'final', + version: setupOptions.version, + isLegacy, }); } @@ -522,14 +565,36 @@ async function main() { let selectedPlugins: string[] = []; let pluginCredentialKeys: string[] = []; - // Use pre-provided plugins from CLI, or prompt for them - if (setupOptions.plugins.length > 0) { - selectedPlugins = setupOptions.plugins; - log.info(`Using plugins: ${selectedPlugins.join(', ')}`); - } else if (!options.full) { - // Only prompt in interactive mode - const availablePlugins = await fetchAvailablePlugins(); - selectedPlugins = await promptPluginSelection(availablePlugins); + if (isLegacy) { + // Legacy SimpleNS <= 1.3.0: plugins always configured + if (setupOptions.plugins.length > 0) { + selectedPlugins = setupOptions.plugins; + log.info(`Using plugins: ${selectedPlugins.join(', ')}`); + } else if (!options.full) { + const availablePlugins = await fetchAvailablePlugins(); + selectedPlugins = await promptPluginSelection(availablePlugins); + } + } else { + // SimpleNS > 1.3.0: plugins are completely optional + if (setupOptions.plugins.length > 0) { + selectedPlugins = setupOptions.plugins; + log.info(`Using plugins: ${selectedPlugins.join(', ')}`); + } else if (setupOptions.skipPlugins || options.full) { + log.info('Skipping plugin configuration. You can install plugins via the Admin Dashboard.'); + } else { + const shouldConfigure = await confirm({ + message: 'Do you want to configure notification plugins now? (Skip to install later via Dashboard)', + initialValue: false, + withGuide: true, + }); + handleCancel(shouldConfigure); + if (shouldConfigure) { + const availablePlugins = await fetchAvailablePlugins(); + selectedPlugins = await promptPluginSelection(availablePlugins); + } else { + log.info('Skipping plugin configuration. You can install plugins via the Admin Dashboard.'); + } + } } if (selectedPlugins.length > 0) { @@ -554,6 +619,24 @@ async function main() { await appendPluginEnv(targetDir, pluginCreds); } } + + // For SimpleNS > 1.3.0, attach simplens.config.yaml mount and SIMPLENS_CONFIG_PATH + if (!isLegacy) { + await writeAppCompose(targetDir, { + includeNginx: false, + includeSsl: false, + isLegacy: false, + hasPluginsConfig: true, + }); + } + } else if (!isLegacy) { + // SimpleNS > 1.3.0 with no plugins selected: ensure compose file has no plugin mounts + await writeAppCompose(targetDir, { + includeNginx: false, + includeSsl: false, + isLegacy: false, + hasPluginsConfig: false, + }); } // Step 6: Service orchestration @@ -600,6 +683,8 @@ async function main() { enableSsl: true, domain: setupOptions.sslDomain, sslMode: 'final', + version: setupOptions.version, + isLegacy, }); await reloadNginxConfiguration(targetDir, { @@ -620,6 +705,8 @@ async function main() { domain: setupOptions.sslDomain, sslMode: 'final', filename: 'nginx.ssl.conf', + version: setupOptions.version, + isLegacy, }); // Print detailed step-by-step SSL setup instructions @@ -655,25 +742,30 @@ async function main() { // In full mode, show a comprehensive security warning if (options.full) { - const credentialWarnings = [ - ' • NS_API_KEY - API authentication key', - ' • AUTH_SECRET - Session secret for dashboard', - ' • ADMIN_PASSWORD - Dashboard admin password', - ]; + const credentialWarnings: string[] = []; + if (isLegacy) { + credentialWarnings.push(' • NS_API_KEY - API authentication key'); + credentialWarnings.push(' • AUTH_SECRET - Session secret for dashboard'); + credentialWarnings.push(' • ADMIN_PASSWORD - Dashboard admin password'); + } else { + credentialWarnings.push(' • JWT_SECRET - Secret key for API authentication tokens'); + } if (pluginCredentialKeys.length > 0) { credentialWarnings.push(` • Plugin credentials: ${pluginCredentialKeys.join(', ')}`); } - note( - '⚠️ IMPORTANT: Auto-generated credentials were used for non-interactive setup.\n' + - '\n' + + let securityNotice = '⚠️ IMPORTANT: Auto-generated credentials were used for non-interactive setup.\n\n' + 'Please update the following in your .env file before production use:\n' + credentialWarnings.join('\n') + '\n\n' + - 'Default credentials are NOT secure for production environments.', - 'Security Notice' - ); + 'Default credentials are NOT secure for production environments.'; + + if (!isLegacy) { + securityNotice += '\n\nAdministrator credentials are set up upon first launch in the Dashboard at /setup.'; + } + + note(securityNotice, 'Security Notice'); } // Display access information @@ -696,6 +788,13 @@ async function main() { ); } + if (!isLegacy) { + note( + 'First-time setup: Open the Dashboard in your browser to configure your administrator account (/setup).', + 'Next Steps' + ); + } + // Clack outro outro('Setup complete — happy notifying! 🚀'); diff --git a/packages/onboard/src/infra.ts b/packages/onboard/src/infra.ts index fc83c8d..8a0a1ef 100644 --- a/packages/onboard/src/infra.ts +++ b/packages/onboard/src/infra.ts @@ -1,5 +1,6 @@ import { APP_COMPOSE_TEMPLATE, + LEGACY_APP_COMPOSE_TEMPLATE, APP_NGINX_SERVICE_TEMPLATE, APP_NGINX_SSL_SERVICE_TEMPLATE, APP_CERTBOT_SERVICES_TEMPLATE, @@ -11,6 +12,7 @@ import { multiselect } from '@clack/prompts'; import { handleCancel, spinner } from './ui.js'; import path from 'path'; import type { InfraService } from './types/domain.js'; +import { isVersionGreaterThan130 } from './env-config.js'; const INFRA_SERVICES: InfraService[] = [ { name: 'MongoDB (Database)', value: 'mongo', checked: true }, @@ -305,15 +307,22 @@ export async function generateInfraCompose( s.stop('Generated docker-compose.infra.yaml'); } +export interface AppComposeOptions { + includeSsl?: boolean; + isLegacy?: boolean; + hasPluginsConfig?: boolean; +} + /** * Build app docker-compose content. * Optionally inject nginx reverse-proxy service before the volumes section. */ export function buildAppComposeContent( includeNginx: boolean, - options: { includeSsl?: boolean } = {} + options: AppComposeOptions = {} ): string { - let content = APP_COMPOSE_TEMPLATE; + const isLegacy = options.isLegacy === true; + let content = isLegacy ? LEGACY_APP_COMPOSE_TEMPLATE : APP_COMPOSE_TEMPLATE; const includeSsl = options.includeSsl === true; const shouldIncludeNginx = includeNginx || includeSsl; const marker = '\nvolumes:'; @@ -322,6 +331,16 @@ export function buildAppComposeContent( return content; } + if (!isLegacy && options.hasPluginsConfig === true) { + const apiMarker = ' api:\n image: ghcr.io/simplenotificationsystem/simplens-core:${VERSION:-latest}\n container_name: api\n ports:\n - 3000:3000\n env_file:\n - .env\n volumes:\n - logs-data:/app/logs'; + const apiReplacement = ' api:\n image: ghcr.io/simplenotificationsystem/simplens-core:${VERSION:-latest}\n container_name: api\n ports:\n - 3000:3000\n env_file:\n - .env\n environment:\n SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml}\n volumes:\n - logs-data:/app/logs\n - ./simplens.config.yaml:/app/simplens.config.yaml:ro'; + content = content.replace(apiMarker, apiReplacement); + + const npMarker = ' notification_processor:\n image: ghcr.io/simplenotificationsystem/simplens-core:${VERSION:-latest}\n container_name: notification_processor\n env_file:\n - .env\n volumes:\n - logs-data:/app/logs'; + const npReplacement = ' notification_processor:\n image: ghcr.io/simplenotificationsystem/simplens-core:${VERSION:-latest}\n container_name: notification_processor\n env_file:\n - .env\n environment:\n SIMPLENS_CONFIG_PATH: ${SIMPLENS_CONFIG_PATH:-/app/simplens.config.yaml}\n volumes:\n - logs-data:/app/logs\n - ./simplens.config.yaml:/app/simplens.config.yaml:ro'; + content = content.replace(npMarker, npReplacement); + } + if (shouldIncludeNginx) { const nginxBlock = includeSsl ? APP_NGINX_SSL_SERVICE_TEMPLATE @@ -331,10 +350,17 @@ export function buildAppComposeContent( if (includeSsl) { content = content.replace(marker, `\n${APP_CERTBOT_SERVICES_TEMPLATE}\n${marker}`); - content = content.replace( - '\nvolumes:\n plugin-data:', - '\nvolumes:\n certbot-etc:\n certbot-www:\n plugin-data:' - ); + if (isLegacy) { + content = content.replace( + '\nvolumes:\n plugin-data:', + '\nvolumes:\n certbot-etc:\n certbot-www:\n plugin-data:' + ); + } else { + content = content.replace( + '\nvolumes:\n logs-data:', + '\nvolumes:\n certbot-etc:\n certbot-www:\n logs-data:' + ); + } } return content; @@ -345,25 +371,41 @@ export function buildAppComposeContent( */ export async function writeAppCompose( targetDir: string, - options: { includeNginx?: boolean; includeSsl?: boolean } = {} + options: { + includeNginx?: boolean; + includeSsl?: boolean; + isLegacy?: boolean; + hasPluginsConfig?: boolean; + } = {} ): Promise { const s = spinner(); s.start('Generating docker-compose.yaml...'); const appPath = path.join(targetDir, 'docker-compose.yaml'); const appContent = buildAppComposeContent(options.includeNginx === true, { includeSsl: options.includeSsl === true, + isLegacy: options.isLegacy === true, + hasPluginsConfig: options.hasPluginsConfig === true, }); await writeFile(appPath, appContent); s.stop('Generated docker-compose.yaml'); } +export interface NginxConfigOptions { + enableSsl?: boolean; + domain?: string; + sslMode?: 'bootstrap' | 'final'; + filename?: string; + version?: string; + isLegacy?: boolean; +} + /** * Generate nginx.conf based on basePath configuration */ export async function generateNginxConfig( targetDir: string, basePath: string, - options: { enableSsl?: boolean; domain?: string; sslMode?: 'bootstrap' | 'final'; filename?: string } = {} + options: NginxConfigOptions = {} ): Promise { const outputFilename = options.filename ?? 'nginx.conf'; const s = spinner(); @@ -375,9 +417,16 @@ export async function generateNginxConfig( const enableSsl = options.enableSsl === true; const domain = options.domain?.trim() || 'localhost'; const sslMode = options.sslMode ?? 'final'; + const isNewVersion = options.isLegacy !== undefined + ? !options.isLegacy + : isVersionGreaterThan130(options.version); + + const apiRoutePrefix = isNewVersion + ? (hasBasePath ? `^~ /${normalizedPath}/api` : '^~ /api') + : '/api'; const proxyRoutes = ` - location /api { + location ${apiRoutePrefix} { proxy_pass http://api:3000; proxy_http_version 1.1; @@ -462,6 +511,7 @@ ${hasBasePath ? ` `; const rootRedirectPath = hasBasePath ? `/${normalizedPath}` : '/'; + const shouldRedirectRoot = isNewVersion && hasBasePath; const nginxTemplate = enableSsl ? sslMode === 'bootstrap' @@ -472,7 +522,10 @@ ${hasBasePath ? ` location /.well-known/acme-challenge/ { root /var/www/certbot; } -${proxyRoutes} +${shouldRedirectRoot ? ` location = / { + return 302 ${rootRedirectPath}; + } +` : ''}${proxyRoutes} } ` : `server { @@ -504,7 +557,11 @@ ${hasBasePath ? ` location = / { ` : `server { listen 80; - server_name localhost;${proxyRoutes} + server_name localhost;${shouldRedirectRoot ? ` + location = / { + return 302 ${rootRedirectPath}; + } +` : ''}${proxyRoutes} } `; diff --git a/packages/onboard/src/templates.ts b/packages/onboard/src/templates.ts index cd7b9d3..5763b8b 100644 --- a/packages/onboard/src/templates.ts +++ b/packages/onboard/src/templates.ts @@ -119,9 +119,9 @@ volumes: grafana_data: `; -export const APP_COMPOSE_TEMPLATE = `services: +export const LEGACY_APP_COMPOSE_TEMPLATE = `services: api: - image: ghcr.io/simplenotificationsystem/simplens-core:\${CORE_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} container_name: api ports: - 3000:3000 @@ -143,7 +143,7 @@ export const APP_COMPOSE_TEMPLATE = `services: start_period: 10s worker: - image: ghcr.io/simplenotificationsystem/simplens-core:\${CORE_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} env_file: - .env volumes: @@ -152,7 +152,7 @@ export const APP_COMPOSE_TEMPLATE = `services: restart: unless-stopped notification_processor: - image: ghcr.io/simplenotificationsystem/simplens-core:\${CORE_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} env_file: - .env environment: @@ -168,7 +168,7 @@ export const APP_COMPOSE_TEMPLATE = `services: restart: unless-stopped delayed_processor: - image: ghcr.io/simplenotificationsystem/simplens-core:\${CORE_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} env_file: - .env volumes: @@ -177,7 +177,7 @@ export const APP_COMPOSE_TEMPLATE = `services: restart: unless-stopped recovery: - image: ghcr.io/simplenotificationsystem/simplens-core:\${CORE_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} env_file: - .env volumes: @@ -186,7 +186,7 @@ export const APP_COMPOSE_TEMPLATE = `services: restart: unless-stopped dashboard: - image: ghcr.io/simplenotificationsystem/simplens-dashboard:\${DASHBOARD_VERSION:-latest} + image: ghcr.io/simplenotificationsystem/simplens-dashboard:\${VERSION:-latest} ports: - 3002:3002 container_name: dashboard @@ -208,6 +208,101 @@ networks: name: simplens `; +export const APP_COMPOSE_TEMPLATE = `services: + api: + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} + container_name: api + ports: + - 3000:3000 + env_file: + - .env + volumes: + - logs-data:/app/logs + command: [ "node", "dist/api/server.js" ] + restart: unless-stopped + healthcheck: + test: [ "CMD", "node", "-e", "require('http').get('http://localhost:3000\${BASE_PATH}/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + worker: + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} + container_name: worker + env_file: + - .env + volumes: + - logs-data:/app/logs + command: [ "node", "dist/workers/worker.js" ] + depends_on: + api: + condition: service_healthy + restart: unless-stopped + + notification_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} + container_name: notification_processor + env_file: + - .env + volumes: + - logs-data:/app/logs + command: [ "node", "dist/processors/unified/unified.processor.js" ] + depends_on: + api: + condition: service_healthy + restart: unless-stopped + + delayed_processor: + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} + container_name: delayed_processor + env_file: + - .env + volumes: + - logs-data:/app/logs + command: [ "node", "dist/processors/delayed/delayed.processor.js" ] + depends_on: + api: + condition: service_healthy + restart: unless-stopped + + recovery: + image: ghcr.io/simplenotificationsystem/simplens-core:\${VERSION:-latest} + container_name: recovery + env_file: + - .env + volumes: + - logs-data:/app/logs + command: [ "node", "dist/workers/recovery/recovery.service.js" ] + depends_on: + api: + condition: service_healthy + restart: unless-stopped + + dashboard: + image: ghcr.io/simplenotificationsystem/simplens-dashboard:\${VERSION:-latest} + ports: + - 3002:3002 + container_name: dashboard + env_file: + - .env + environment: + PORT: \${DASHBOARD_PORT:-3002} + BASE_PATH: \${BASE_PATH:-} + API_BASE_URL: \${API_BASE_URL:-http://api:\${PORT:-3000}} + depends_on: + api: + condition: service_healthy + restart: unless-stopped + +volumes: + logs-data: + +networks: + default: + name: simplens +`; + export const APP_NGINX_SERVICE_TEMPLATE = ` nginx: image: nginx:alpine container_name: nginx diff --git a/scripts/README.md b/scripts/README.md index f22ccca..65ab842 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,14 +1,11 @@ # SimpleNS Testing & Utility Scripts -This directory contains developer utilities for testing, load generation, webhook verification, and resilience/chaos testing for SimpleNS. +This directory contains developer utilities for testing, load generation, webhook verification for SimpleNS. ``` scripts/ ├── client.js # Lightweight Express webhook receiver for callback inspection ├── load-test.js # Universal notification sender & high-concurrency load tester -├── crash-tests/ -│ ├── k8s-chaos.ps1 # Kubernetes-native container & pod chaos testing -│ └── crash-multiwave-all.sh # Docker Compose infrastructure chaos testing (8 waves) └── README.md # This documentation ``` @@ -116,95 +113,4 @@ node scripts/load-test.js -n 1000 -c 50 -ch mock -p mock -h docker ``` *Reports throughput (req/sec), min/avg/max latency, `p50`, `p95`, `p99` percentiles, and HTTP status code distributions.* ---- - -## 3. Chaos & Resilience Testing (`crash-tests/`) - -Simulate service crashes, database outages, connection drops, and container evictions under active load to verify outbox recovery and idempotency. - -### A. Kubernetes Native Chaos (`k8s-chaos.ps1`) - -Designed specifically for Kubernetes clusters (Kind or Docker Desktop) running the `app-local` and `simplens-infra` pods. - -Instead of crashing Docker directly, it: -1. Generates active background notification traffic using `load-test.js`. -2. Kills specific container processes inside pods via `kubectl exec ... kill 1`. -3. Evicts whole pods via `kubectl delete pod --now`. -4. Disrupts infrastructure (Redis, Kafka) inside the infra pod. -5. Verifies self-healing and outbox recovery. - -#### Usage - -```powershell -# Run all 4 chaos waves with 300 requests: -./scripts/crash-tests/k8s-chaos.ps1 - -# Custom request volume: -./scripts/crash-tests/k8s-chaos.ps1 -Requests 500 - -# Target a specific wave: -./scripts/crash-tests/k8s-chaos.ps1 -Wave app # Individual container kills (worker, processor, delayed) -./scripts/crash-tests/k8s-chaos.ps1 -Wave infra # Redis & Kafka disruptions -./scripts/crash-tests/k8s-chaos.ps1 -Wave pod # Full app-local pod eviction -./scripts/crash-tests/k8s-chaos.ps1 -Wave cascade # Simultaneous app + infra collapse -``` - ---- - -### B. Docker Compose Chaos (`crash-multiwave-all.sh`) - -Designed for environments running directly via `docker compose -f docker-compose.dev.yaml up -d`. - -Simulates 8 distinct disaster waves: -1. **Sustained Load Crash**: Sequential app + Redis kills under heavy load. -2. **Rapid Restart Chaos**: 5 quick crash-restart cycles testing connection pools. -3. **Split Brain Attack**: Partial service availability testing orphan recovery. -4. **Total Blackout**: Complete simultaneous death of app + infra services. -5. **Rolling Chaos**: Continuous random service kills over 60 seconds. -6. **Infrastructure Chaos**: Targeted Redis, Kafka, and MongoDB crashes. -7. **Database Partition**: MongoDB replica crashes during active transactions. -8. **Complete Apocalypse**: Everything killed at once, random recovery race. - -#### Usage (Git Bash / Linux / macOS) - -```bash -cd scripts/crash-tests - -# Run all 8 waves with 500 requests per wave and 30s pause: -./crash-multiwave-all.sh 500 30 -``` - ---- - -## 4. End-to-End Verification Workflow - -To test the entire system end-to-end against a running cluster: - -### Step 1: Start the Webhook Server -In Terminal 1: -```powershell -node scripts/client.js -``` - -### Step 2: Ensure Port Forwarding or NodePorts are Active -In Terminal 2: -```powershell -./k8s/scripts/win/port-forward.ps1 -``` -*(Or use direct Kind NodePorts `http://localhost:30300` for API and `http://localhost:30302` for Dashboard).* - -### Step 3: Run the Chaos Test -In Terminal 3: -```powershell -./scripts/crash-tests/k8s-chaos.ps1 -Requests 300 -``` - -### Step 4: Verify Delivery -- In Terminal 1, watch the incoming webhook deliveries logged. -- Check stats: `curl http://localhost:4000/stats` -- Check Dashboard Alerts: [http://localhost:3002](http://localhost:3002) -- Check Recovery Service Logs: - ```powershell - kubectl logs -n simplens deploy/app-local -c recovery -f - ``` - +--- \ No newline at end of file diff --git a/scripts/crash-tests/crash-multiwave-all.sh b/scripts/crash-tests/crash-multiwave-all.sh deleted file mode 100644 index 341d123..0000000 --- a/scripts/crash-tests/crash-multiwave-all.sh +++ /dev/null @@ -1,843 +0,0 @@ -#!/bin/bash -# ============================================================================== -# Multi-Wave Chaos Crash Test - COMPLETE INFRASTRUCTURE EDITION -# ============================================================================== -# Simulates multiple aggressive waves of crashes on ALL services including -# infrastructure (MongoDB, Redis, Kafka) to maximally stress the notification -# system's recovery mechanisms. -# -# Wave Types: -# 1. Sustained Load Wave - Crashes app services under constant high load -# 2. Rapid Restart Wave - Quick crash-restart cycles (app + infra) -# 3. Split Brain Wave - Processors up/down in patterns -# 4. Total Blackout Wave - EVERYTHING down (incl. infra), staggered recovery -# 5. Rolling Chaos Wave - Random service kills with load -# 6. Infrastructure Chaos Wave - Database, Redis, Kafka crashes -# 7. Database Partition Wave - MongoDB crash and recovery patterns -# 8. Complete Apocalypse Wave - Everything simultaneously, recovery race -# -# Usage: ./crash-multiwave-all.sh [load_requests] [pause_between_waves] -# Example: ./crash-multiwave-all.sh 500 30 -# ============================================================================== - -set -e - -# Configuration -LOAD_REQUESTS=${1:-500} -WAVE_PAUSE=${2:-30} -SCRIPTS_DIR=$(cd "$(dirname "$0")/.." && pwd) -PROJECT_ROOT=$(cd "$(dirname "$0")/../.." && pwd) - -# Determine active compose file -COMPOSE_FILE="$PROJECT_ROOT/docker-compose.dev.yaml" -if [ ! -f "$COMPOSE_FILE" ]; then - COMPOSE_FILE="$PROJECT_ROOT/docker-compose.yaml" -fi - -# Application and Infrastructure Services -# Recovery service is intentionally excluded from kills to keep it monitoring and recovering -APP_SERVICES=( - "worker" - "notification-processor" - "delayed-processor" -) - -INFRA_SERVICES=( - "mongo" - "redis" - "kafka" -) - -# Service identifiers used across chaos waves -APP_CONTAINERS=("${APP_SERVICES[@]}") -INFRA_CONTAINERS=("${INFRA_SERVICES[@]}") -CONTAINERS=("${APP_CONTAINERS[@]}" "${INFRA_CONTAINERS[@]}") -SERVICES=("${APP_SERVICES[@]}" "${INFRA_SERVICES[@]}") - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -MAGENTA='\033[0;35m' -CYAN='\033[0;36m' -NC='\033[0m' # No Color - -# Helper functions -print_header() { - echo "" - echo -e "${MAGENTA}════════════════════════════════════════════════════════════════${NC}" - echo -e "${MAGENTA} $1${NC}" - echo -e "${MAGENTA}════════════════════════════════════════════════════════════════${NC}" - echo "" -} - -print_wave() { - echo "" - echo -e "${CYAN}┌──────────────────────────────────────────────────────────────┐${NC}" - echo -e "${CYAN}│ 🌊 WAVE $1: $2${NC}" - echo -e "${CYAN}└──────────────────────────────────────────────────────────────┘${NC}" - echo "" -} - -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $1" -} - -log_crash() { - echo -e "${RED}[CRASH]${NC} 💥 $1" -} - -log_recover() { - echo -e "${GREEN}[RECOVER]${NC} ✅ $1" -} - -# Kill a service or container cleanly -kill_container() { - local target="$1" - # Try docker compose kill by service name first - if docker compose -f "$COMPOSE_FILE" kill "$target" 2>/dev/null; then - return 0 - fi - # Fallback to direct container kill by name/id - docker kill "$target" 2>/dev/null || true -} - -# Kill multiple containers/services -kill_containers() { - for container in "$@"; do - kill_container "$container" - log_crash "$container" - done -} - -# Restart all application services -restart_all_app_services() { - log_recover "Restarting all application services..." - docker compose -f "$COMPOSE_FILE" up -d "${APP_SERVICES[@]}" - sleep 3 -} - -# Restart all infrastructure services -restart_all_infra_services() { - log_recover "Restarting all infrastructure services..." - docker compose -f "$COMPOSE_FILE" up -d "${INFRA_SERVICES[@]}" - sleep 5 - wait_for_infra_health -} - -# Restart all services (infra first, then app) -restart_all_services() { - log_recover "Restarting all services..." - docker compose -f "$COMPOSE_FILE" up -d "${INFRA_SERVICES[@]}" - sleep 5 - wait_for_infra_health - docker compose -f "$COMPOSE_FILE" up -d "${APP_SERVICES[@]}" - sleep 3 -} - -# Restart specific services -restart_services() { - for service in "$@"; do - log_recover "$service" - docker compose -f "$COMPOSE_FILE" up -d "$service" - done -} - -# Wait for infrastructure to be healthy -wait_for_infra_health() { - log_info "Waiting for infrastructure services to be healthy..." - local max_wait=60 - local waited=0 - - while [ $waited -lt $max_wait ]; do - # Check each service using exit-code based checks - local mongo_ok=0 - local redis_ok=0 - local kafka_ok=0 - - # MongoDB check (try compose exec first, fallback to docker exec) - if docker compose -f "$COMPOSE_FILE" exec -T mongo mongosh --eval "db.adminCommand('ping')" >/dev/null 2>&1 || docker exec mongo mongosh --eval "db.adminCommand('ping')" >/dev/null 2>&1; then - mongo_ok=1 - fi - - # Redis check - if docker compose -f "$COMPOSE_FILE" exec -T redis redis-cli ping 2>/dev/null | grep -q "PONG" || docker exec redis redis-cli ping 2>/dev/null | grep -q "PONG"; then - redis_ok=1 - fi - - # Kafka check - if docker compose -f "$COMPOSE_FILE" exec -T kafka kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1 || docker exec kafka kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1; then - kafka_ok=1 - fi - - if [ $mongo_ok -eq 1 ] && [ $redis_ok -eq 1 ] && [ $kafka_ok -eq 1 ]; then - log_info "All infrastructure services healthy!" - return 0 - fi - - sleep 2 - waited=$((waited + 2)) - done - - log_warn "Timeout waiting for infrastructure health, continuing anyway..." -} - -# Start load test in background -start_load() { - local requests=$1 - log_info "Starting load test with $requests requests..." - cd "$PROJECT_ROOT" - node scripts/load-test.js -n "$requests" -c 50 -ch mock -p mock -m "Chaos test message" -h docker & - LOAD_PID=$! - sleep 2 # Give load test time to start -} - -# Wait for load to finish -wait_for_load() { - if [ -n "$LOAD_PID" ]; then - log_info "Waiting for load test to complete..." - wait $LOAD_PID 2>/dev/null || true - fi -} - -# Random delay between min and max seconds -random_delay() { - local min=$1 - local max=$2 - local delay=$((RANDOM % (max - min + 1) + min)) - sleep $delay -} - -# Get random container(s) from all containers -random_containers() { - local count=$1 - local shuffled=($(shuf -e "${CONTAINERS[@]}")) - echo "${shuffled[@]:0:$count}" -} - -# Get random app container(s) -random_app_containers() { - local count=$1 - local shuffled=($(shuf -e "${APP_CONTAINERS[@]}")) - echo "${shuffled[@]:0:$count}" -} - -# Get random infra container(s) -random_infra_containers() { - local count=$1 - local shuffled=($(shuf -e "${INFRA_CONTAINERS[@]}")) - echo "${shuffled[@]:0:$count}" -} - -# ============================================================================== -# WAVE 1: Sustained Load Crash -# Crashes services while under constant high load -# ============================================================================== -wave_sustained_load() { - print_wave "1/8" "SUSTAINED LOAD CRASH" - log_info "Strategy: Crash app services + Redis sequentially while under constant load" - log_info "This tests message loss and recovery under production-like conditions" - echo "" - - # Start sustained load - start_load $LOAD_REQUESTS - - # Crash services one by one with small delays - sleep 5 - log_crash "Killing notification-processor mid-processing..." - kill_container "${APP_CONTAINERS[1]}" - - sleep 3 - log_crash "Killing worker mid-outbox-poll..." - kill_container "${APP_CONTAINERS[0]}" - - sleep 3 - log_crash "Killing Redis - cache disruption..." - kill_container "redis" - - # Let the chaos continue for a bit - sleep 10 - - # Restart in wrong order (processors before worker, but infra first) - log_info "Restarting infrastructure first..." - restart_services "redis" - sleep 5 - wait_for_infra_health - - log_info "Restarting in wrong order to create backpressure..." - restart_services "notification-processor" - sleep 5 - restart_services "worker" - - wait_for_load - log_info "Wave 1 complete - allowing recovery service to detect issues..." - sleep 15 -} - -# ============================================================================== -# WAVE 2: Rapid Restart Chaos -# Quick crash-restart cycles to stress connection handling -# ============================================================================== -wave_rapid_restart() { - print_wave "2/8" "RAPID RESTART CHAOS" - log_info "Strategy: Rapid crash-restart cycles to stress connection pools" - log_info "This tests Kafka consumer group rebalancing and Redis reconnection" - echo "" - - local cycles=5 - - for i in $(seq 1 $cycles); do - echo "" - log_info "Rapid cycle $i/$cycles" - - # Crash random service - local container=$(random_containers 1) - log_crash "$container" - kill_container "$container" - - # Very short delay - sleep 2 - - # Crash another - container=$(random_containers 1) - log_crash "$container" - kill_container "$container" - - # Quick restart - sleep 3 - restart_all_services - - # Tiny stabilization - sleep 2 - done - - log_info "Wave 2 complete - services may be in unstable state..." - sleep 10 -} - -# ============================================================================== -# WAVE 3: Split Brain Attack -# Create scenarios where some processors are up and others down -# ============================================================================== -wave_split_brain() { - print_wave "3/8" "SPLIT BRAIN ATTACK" - log_info "Strategy: Create inconsistent states with partial service availability" - log_info "This tests status reconciliation and orphaned message detection" - echo "" - - # Start load - start_load $((LOAD_REQUESTS / 2)) - - # Pattern 1: Only notification processor works - log_info "Pattern 1: Only notification processor alive..." - kill_containers "${APP_CONTAINERS[0]}" "${APP_CONTAINERS[2]}" - sleep 15 - restart_all_app_services - sleep 5 - - # Pattern 2: Only delayed processor works - log_info "Pattern 2: Only delayed processor alive..." - kill_containers "${APP_CONTAINERS[0]}" "${APP_CONTAINERS[1]}" - sleep 15 - restart_all_app_services - sleep 5 - - # Pattern 3: Worker down, all processors up (messages accepted but not routed) - log_info "Pattern 3: Worker down - messages queued but not routed..." - kill_container "${APP_CONTAINERS[0]}" - sleep 20 - restart_services "worker" - - wait_for_load - log_info "Wave 3 complete - checking for split brain artifacts..." - sleep 15 -} - -# ============================================================================== -# WAVE 4: Total Blackout -# Everything dies (including infra), then comes back in various orders -# ============================================================================== -wave_total_blackout() { - print_wave "4/8" "TOTAL BLACKOUT" - log_info "Strategy: Complete infrastructure failure including MongoDB, Redis, Kafka" - log_info "This tests system boot order dependencies and data persistence" - echo "" - - # Start load first - start_load $((LOAD_REQUESTS / 3)) - sleep 5 - - # BLACKOUT! - echo "" - echo -e "${RED}╔════════════════════════════════════════════════════════════════╗${NC}" - echo -e "${RED}║ ⚡ TOTAL INFRASTRUCTURE BLACKOUT INITIATED ⚡ ║${NC}" - echo -e "${RED}║ Including: MongoDB, Redis, Kafka, All Processors ║${NC}" - echo -e "${RED}╚════════════════════════════════════════════════════════════════╝${NC}" - echo "" - - # Kill app services first - log_crash "Killing all application services..." - kill_containers "${APP_CONTAINERS[@]}" - - sleep 5 - - # Then infrastructure - this is brutal - log_crash "Killing Redis..." - kill_container "redis" - - log_crash "Killing Kafka..." - kill_container "kafka" - - log_crash "Killing MongoDB..." - kill_container "mongo" - - # Let everything be dead - log_info "Complete infrastructure blackout - everything is dead..." - sleep 20 - - wait_for_load - - # Recovery in proper order - infrastructure first - echo "" - log_info "Beginning staged recovery..." - - log_info "Stage 1: Bringing MongoDB online first (data layer)..." - restart_services "mongo" - sleep 10 - - log_info "Stage 2: Bringing Redis online (cache layer)..." - restart_services "redis" - sleep 5 - - log_info "Stage 3: Bringing Kafka online (message bus)..." - restart_services "kafka" - sleep 10 - - # Wait for infrastructure health - wait_for_infra_health - - log_info "Stage 4: Bringing delayed processor online..." - restart_services "delayed-processor" - sleep 5 - - log_info "Stage 5: Bringing worker online..." - restart_services "worker" - sleep 5 - - log_info "Stage 6: Bringing notification processor online..." - restart_services "notification-processor" - - log_info "Wave 4 complete - system recovering from total blackout..." - sleep 20 -} - -# ============================================================================== -# WAVE 5: Rolling Chaos -# Continuous random crashes including infrastructure with concurrent load -# ============================================================================== -wave_rolling_chaos() { - print_wave "5/8" "ROLLING CHAOS" - log_info "Strategy: Unpredictable random crashes including infrastructure" - log_info "This is a major stress test - pure chaos across all services!" - echo "" - - # Start continuous load - start_load $LOAD_REQUESTS - - local duration=60 # seconds of chaos - local start_time=$(date +%s) - local crash_count=0 - - log_info "Running $duration seconds of pure chaos..." - echo "" - - while true; do - local current_time=$(date +%s) - local elapsed=$((current_time - start_time)) - - if [ $elapsed -ge $duration ]; then - break - fi - - # Random action - expanded to include infra - local action=$((RANDOM % 6)) - - case $action in - 0) - # Kill random app service - local container=$(random_app_containers 1) - log_crash "$container (random app kill)" - kill_container "$container" - ;; - 1) - # Kill two random app services - local containers=($(random_app_containers 2)) - for c in "${containers[@]}"; do - log_crash "$c (double app kill)" - kill_container "$c" - done - ;; - 2) - # Kill all app services then restart one - log_crash "ALL APP SERVICES (chaos burst)" - kill_containers "${APP_CONTAINERS[@]}" - sleep 2 - local service=${APP_SERVICES[$((RANDOM % ${#APP_SERVICES[@]}))]} - log_recover "$service (single survivor)" - restart_services "$service" - ;; - 3) - # Quick restart cycle - local container=$(random_app_containers 1) - log_crash "$container (restart cycle)" - kill_container "$container" - sleep 1 - restart_all_app_services - ;; - 4) - # Kill Redis briefly - log_crash "redis (brief infra kill)" - kill_container "redis" - sleep 3 - restart_services "redis" - ;; - 5) - # Kill random infrastructure service - local infra_container=$(random_infra_containers 1) - log_crash "$infra_container (infra chaos)" - kill_container "$infra_container" - sleep 5 - restart_all_infra_services - ;; - esac - - crash_count=$((crash_count + 1)) - random_delay 3 8 - done - - echo "" - log_info "Chaos complete! Total crash events: $crash_count" - - # Final recovery - restart_all_services - wait_for_load - - log_info "Wave 5 complete - final recovery initiated..." - sleep 15 -} - -# ============================================================================== -# WAVE 6: Infrastructure Chaos -# Specifically targets MongoDB, Redis, and Kafka -# ============================================================================== -wave_infrastructure_chaos() { - print_wave "6/8" "INFRASTRUCTURE CHAOS" - log_info "Strategy: Targeted attacks on core infrastructure components" - log_info "This tests data layer resilience and connection recovery" - echo "" - - # Pattern 1: Redis crash while processing - log_info "Pattern 1: Redis crash during active processing..." - start_load $((LOAD_REQUESTS / 4)) - sleep 5 - log_crash "Killing Redis..." - kill_container "redis" - sleep 15 - log_recover "Restarting Redis..." - restart_services "redis" - wait_for_load - sleep 10 - - # Pattern 2: Kafka crash - message broker down - log_info "Pattern 2: Kafka crash - message broker failure..." - start_load $((LOAD_REQUESTS / 4)) - sleep 5 - log_crash "Killing Kafka..." - kill_container "kafka" - sleep 20 # Kafka takes longer to recover - log_recover "Restarting Kafka..." - restart_services "kafka" - sleep 15 - wait_for_load - sleep 10 - - # Pattern 3: MongoDB crash - primary database failure - log_info "Pattern 3: MongoDB crash - database failure..." - start_load $((LOAD_REQUESTS / 4)) - sleep 5 - log_crash "Killing MongoDB..." - kill_container "mongo" - sleep 20 - log_recover "Restarting MongoDB..." - restart_services "mongo" - sleep 15 - wait_for_infra_health - wait_for_load - sleep 10 - - # Pattern 4: Cascading infrastructure failure - log_info "Pattern 4: Cascading infrastructure failure..." - start_load $((LOAD_REQUESTS / 4)) - sleep 5 - - log_crash "Redis down..." - kill_container "redis" - sleep 3 - log_crash "Kafka down..." - kill_container "kafka" - sleep 15 - - # Recovery in reverse order - log_recover "Kafka recovering..." - restart_services "kafka" - sleep 5 - log_recover "Redis recovering..." - restart_services "redis" - - wait_for_infra_health - wait_for_load - - log_info "Wave 6 complete - infrastructure chaos survived!" - sleep 15 -} - -# ============================================================================== -# WAVE 7: Database Partition -# MongoDB specific crashes to test data consistency -# ============================================================================== -wave_database_partition() { - print_wave "7/8" "DATABASE PARTITION" - log_info "Strategy: MongoDB crash patterns to test data consistency" - log_info "This tests transaction handling and outbox pattern resilience" - echo "" - - # Pattern 1: DB crash during outbox poll - log_info "Pattern 1: MongoDB crash during outbox poll..." - start_load $((LOAD_REQUESTS / 3)) - sleep 10 # Let messages start processing - - log_crash "MongoDB crash mid-transaction..." - kill_container "mongo" - - # Workers should be stuck, messages should be in various states - sleep 20 - - log_recover "MongoDB recovering..." - restart_services "mongo" - sleep 10 - wait_for_infra_health - wait_for_load - sleep 10 - - # Pattern 2: Rapid MongoDB restart - log_info "Pattern 2: Rapid MongoDB restart cycles..." - for i in $(seq 1 3); do - log_info "MongoDB restart cycle $i/3" - log_crash "MongoDB down..." - kill_container "mongo" - sleep 5 - log_recover "MongoDB up..." - restart_services "mongo" - sleep 10 - done - wait_for_infra_health - sleep 10 - - # Pattern 3: DB + App service coordinated crash - log_info "Pattern 3: Coordinated DB and worker crash..." - start_load $((LOAD_REQUESTS / 3)) - sleep 5 - - log_crash "Worker + MongoDB simultaneous crash..." - kill_container "${APP_CONTAINERS[0]}" - kill_container "mongo" - sleep 15 - - log_recover "MongoDB first, then worker..." - restart_services "mongo" - sleep 10 - wait_for_infra_health - restart_services "worker" - - wait_for_load - log_info "Wave 7 complete - database partition test finished!" - sleep 15 -} - -# ============================================================================== -# WAVE 8: Complete Apocalypse -# Everything crashes simultaneously, system races to recover -# ============================================================================== -wave_complete_apocalypse() { - print_wave "8/8" "COMPLETE APOCALYPSE" - log_info "Strategy: Simultaneous crash of EVERYTHING - the ultimate test" - log_info "This tests total system recovery from catastrophic failure" - echo "" - - # Start heavy load - start_load $LOAD_REQUESTS - sleep 10 # Let processing begin - - echo "" - echo -e "${RED}╔════════════════════════════════════════════════════════════════╗${NC}" - echo -e "${RED}║ ║${NC}" - echo -e "${RED}║ 💀 COMPLETE APOCALYPSE - KILLING EVERYTHING 💀 ║${NC}" - echo -e "${RED}║ ║${NC}" - echo -e "${RED}║ MongoDB 🔴 Redis 🔴 Kafka 🔴 ║${NC}" - echo -e "${RED}║ Worker 🔴 Notification 🔴 Delayed 🔴 ║${NC}" - echo -e "${RED}║ ║${NC}" - echo -e "${RED}╚════════════════════════════════════════════════════════════════╝${NC}" - echo "" - - # Kill EVERYTHING at once - log_crash "Killing ALL services simultaneously..." - for container in "${CONTAINERS[@]}"; do - kill_container "$container" & - done - wait # Wait for all kills to complete - - # Total darkness - log_info "Complete system failure - all services dead..." - sleep 30 - - wait_for_load - - # Chaotic recovery - random order - echo "" - log_info "Beginning chaotic recovery..." - - # Shuffle and recover in random order (but infrastructure first) - log_info "Recovering infrastructure in random order..." - local shuffled_infra=($(shuf -e "${INFRA_SERVICES[@]}")) - for service in "${shuffled_infra[@]}"; do - log_recover "$service (random infra recovery)" - restart_services "$service" - sleep 5 - done - - wait_for_infra_health - - log_info "Recovering app services in random order..." - local shuffled_app=($(shuf -e "${APP_SERVICES[@]}")) - for service in "${shuffled_app[@]}"; do - log_recover "$service (random app recovery)" - restart_services "$service" - sleep 3 - done - - log_info "Wave 8 complete - survived the apocalypse!" - sleep 20 -} - -# ============================================================================== -# Main Execution -# ============================================================================== -main() { - print_header "🔥 MULTI-WAVE CHAOS CRASH TEST - COMPLETE INFRASTRUCTURE EDITION 🔥" - - echo "Configuration:" - echo " Load Requests per Wave: $LOAD_REQUESTS" - echo " Pause Between Waves: ${WAVE_PAUSE}s" - echo " Project Root: $PROJECT_ROOT" - echo "" - echo "Services to be crashed:" - echo " Application: worker, notification-processor (unified), delayed-processor" - echo " Infrastructure: MongoDB, Redis, Kafka" - echo "" - - log_warn "⚠️ This test WILL crash ALL your services including infrastructure!" - log_warn "⚠️ MongoDB, Redis, and Kafka WILL be killed during this test!" - log_warn "Make sure recovery service is running: docker logs -f ns-recovery" - echo "" - log_info "Starting in 5 seconds... (Ctrl+C to abort)" - sleep 5 - - # Ensure all services are running first - log_info "Ensuring all services are running..." - restart_all_services - sleep 5 - - # Execute waves - wave_sustained_load - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_rapid_restart - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_split_brain - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_total_blackout - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_rolling_chaos - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_infrastructure_chaos - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before next wave..." - sleep $WAVE_PAUSE - - wave_database_partition - echo "" - log_info "⏸️ Pause for ${WAVE_PAUSE}s before final wave..." - sleep $WAVE_PAUSE - - wave_complete_apocalypse - - # Final summary - print_header "🏁 COMPLETE INFRASTRUCTURE CHAOS TEST COMPLETE 🏁" - - echo "All 8 waves completed! Your system survived:" - echo " ✓ Application service crashes" - echo " ✓ Infrastructure crashes (MongoDB, Redis, Kafka)" - echo " ✓ Complete blackouts" - echo " ✓ Database partitions" - echo " ✓ The apocalypse" - echo "" - echo "Review the following:" - echo "" - echo "1. Recovery Service Logs:" - echo " docker compose -f \"$COMPOSE_FILE\" logs -f recovery" - echo "" - echo "2. Dashboard Alerts:" - echo " http://localhost:3002" - echo "" - echo "3. Check for stuck notifications:" - echo " docker compose -f \"$COMPOSE_FILE\" exec -T mongo mongosh --eval \"use notification_service; db.notifications.countDocuments({status: 'processing'})\"" - echo "" - echo "4. Check Kafka consumer lag:" - echo " docker compose -f \"$COMPOSE_FILE\" exec -T kafka kafka-broker-api-versions.sh --bootstrap-server localhost:9092" - echo "" - echo "5. Check Redis connectivity:" - echo " docker compose -f \"$COMPOSE_FILE\" exec -T redis redis-cli ping" - echo "" - echo "6. Check MongoDB replica set status:" - echo " docker compose -f \"$COMPOSE_FILE\" exec -T mongo mongosh --eval \"rs.status()\"" - echo "" - - log_info "Final service restart..." - restart_all_services - - echo "" - log_info "✅ All services restored. Monitor recovery for the next few minutes." - log_info "🎉 Congratulations! Your system survived complete infrastructure chaos!" -} - -# Run main -main "$@" diff --git a/scripts/crash-tests/k8s-chaos.ps1 b/scripts/crash-tests/k8s-chaos.ps1 deleted file mode 100644 index 39aac59..0000000 --- a/scripts/crash-tests/k8s-chaos.ps1 +++ /dev/null @@ -1,159 +0,0 @@ -<# -.SYNOPSIS - Kubernetes Chaos & Crash Test Script for SimpleNS - Simulates container failures, pod evictions, and infrastructure disruptions - against running Kubernetes pods (app-local and simplens-infra). - -.PARAMETER Requests - Number of background test requests to send during each chaos wave (default: 300). - -.PARAMETER Wave - Specific wave to run: 'all', 'app', 'infra', 'pod', 'cascade' (default: 'all'). -#> -param( - [int]$Requests = 300, - [ValidateSet("all", "app", "infra", "pod", "cascade")] - [string]$Wave = "all" -) - -$ErrorActionPreference = "Continue" -$Namespace = "simplens" -$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path -$RootDir = Split-Path -Parent (Split-Path -Parent $ScriptDir) - -function Log-Info($msg) { Write-Host "[INFO] $msg" -ForegroundColor Cyan } -function Log-Crash($msg) { Write-Host "[CRASH] $msg" -ForegroundColor Red } -function Log-Recover($msg) { Write-Host "[RECOVER] $msg" -ForegroundColor Green } -function Log-Wave($num, $title) { - Write-Host "`n================================================================" -ForegroundColor Magenta - Write-Host " [WAVE $num] $title" -ForegroundColor Magenta - Write-Host "================================================================`n" -ForegroundColor Magenta -} - -# Start background load using load-test.js -function Start-BackgroundLoad([int]$count) { - Log-Info "Starting background load ($count requests) against API..." - $job = Start-Job -ScriptBlock { - param($root, $n) - Set-Location $root - node "$root/scripts/load-test.js" -n $n -c 25 -ch mock -p mock -h docker - } -ArgumentList $RootDir, $count - Start-Sleep -Seconds 2 - return $job -} - -# 1. Kill a specific container inside app-local pod -function Kill-AppContainer([string]$containerName) { - Log-Crash "Killing container '$containerName' in pod 'app-local'..." - kubectl exec -n $Namespace deploy/app-local -c $containerName -- kill 1 2>$null -} - -# 2. Kill a specific container inside simplens-infra pod -function Kill-InfraContainer([string]$containerName) { - Log-Crash "Killing container '$containerName' in pod 'simplens-infra'..." - if ($containerName -eq "redis") { - kubectl exec -n $Namespace deploy/simplens-infra -c redis -- redis-cli shutdown nosave 2>$null - } elseif ($containerName -eq "mongo") { - kubectl exec -n $Namespace deploy/simplens-infra -c mongo -- mongosh --eval "db.adminCommand({shutdown: 1, force: true})" 2>$null - } else { - kubectl exec -n $Namespace deploy/simplens-infra -c $containerName -- kill 1 2>$null - } -} - -# 3. Wait for app pod ready -function Wait-AppReady { - Log-Info "Waiting for app-local pod to return to Ready state..." - kubectl wait --namespace $Namespace --for=condition=ready pod -l app.kubernetes.io/name=app-local --timeout=60s 2>$null -} - -# 4. Wait for infra pod ready -function Wait-InfraReady { - Log-Info "Waiting for simplens-infra pod to return to Ready state..." - kubectl wait --namespace $Namespace --for=condition=ready pod -l app.kubernetes.io/name=simplens-infra --timeout=90s 2>$null -} - -Write-Host "================================================================" -ForegroundColor Yellow -Write-Host " SimpleNS Kubernetes Native Chaos & Resilience Test" -ForegroundColor Yellow -Write-Host " Target Cluster: Kind | Namespace: $Namespace" -ForegroundColor Yellow -Write-Host "================================================================" -ForegroundColor Yellow - -# Wave 1: Application Container Failures -if ($Wave -eq "all" -or $Wave -eq "app") { - Log-Wave "1" "INDIVIDUAL APP CONTAINER CRASHES UNDER LOAD" - $loadJob = Start-BackgroundLoad $Requests - - Start-Sleep -Seconds 3 - Kill-AppContainer "notification-processor" - Start-Sleep -Seconds 5 - - Kill-AppContainer "worker" - Start-Sleep -Seconds 5 - - Kill-AppContainer "delayed-processor" - - Log-Info "Waiting for containers to self-heal and load to complete..." - Wait-AppReady - $loadJob | Wait-Job | Receive-Job - $loadJob | Remove-Job -Force -} - -# Wave 2: Infrastructure Disruptions (Kafka / Redis) -if ($Wave -eq "all" -or $Wave -eq "infra") { - Log-Wave "2" "INFRASTRUCTURE DISRUPTIONS (Redis & Kafka)" - $loadJob = Start-BackgroundLoad $Requests - - Start-Sleep -Seconds 3 - Kill-InfraContainer "redis" - Start-Sleep -Seconds 6 - - Kill-InfraContainer "kafka" - Start-Sleep -Seconds 8 - - Wait-InfraReady - Wait-AppReady - $loadJob | Wait-Job | Receive-Job - $loadJob | Remove-Job -Force -} - -# Wave 3: Entire Pod Eviction / Hard Kill -if ($Wave -eq "all" -or $Wave -eq "pod") { - Log-Wave "3" "FULL APP-LOCAL POD EVICTION" - $loadJob = Start-BackgroundLoad $Requests - - Start-Sleep -Seconds 3 - Log-Crash "Force-deleting the entire 'app-local' pod!" - kubectl delete pod -n $Namespace -l app.kubernetes.io/name=app-local --now 2>$null - - Log-Info "Kubelet is recreating the pod..." - Wait-AppReady - $loadJob | Wait-Job | Receive-Job - $loadJob | Remove-Job -Force -} - -# Wave 4: Cascading Failure -if ($Wave -eq "all" -or $Wave -eq "cascade") { - Log-Wave "4" "CASCADING CHAOS (Infra + App simultaneous death)" - $loadJob = Start-BackgroundLoad $Requests - - Start-Sleep -Seconds 3 - Kill-InfraContainer "redis" - Kill-AppContainer "worker" - Kill-AppContainer "notification-processor" - - Start-Sleep -Seconds 6 - Log-Crash "Evicting app pod while Redis is recovering..." - kubectl delete pod -n $Namespace -l app.kubernetes.io/name=app-local --now 2>$null - - Wait-InfraReady - Wait-AppReady - $loadJob | Wait-Job | Receive-Job - $loadJob | Remove-Job -Force -} - -Write-Host "`n================================================================" -ForegroundColor Green -Write-Host " [OK] Kubernetes Chaos Test Complete!" -ForegroundColor Green -Write-Host "================================================================" -ForegroundColor Green -Write-Host "Verify system status:" -ForegroundColor Cyan -Write-Host " - Pod status: kubectl get pods -n $Namespace" -ForegroundColor White -Write-Host " - Recovery Logs: kubectl logs -n $Namespace deploy/app-local -c recovery" -ForegroundColor White -Write-Host " - Dashboard Alerts: http://localhost:3002" -ForegroundColor White diff --git a/scripts/load-test.js b/scripts/load-test.js index 59684e8..fea9716 100644 --- a/scripts/load-test.js +++ b/scripts/load-test.js @@ -17,7 +17,7 @@ * -s, --subject Subject (for email) * -m, --message Message content (for all channels) * -u, --url Base API URL (default: http://localhost:3000) - * -k, --key SimpleNS API Key (defaults to NS_API_KEY env or dev key) + * -k, --key SimpleNS API Key (defaults to API_KEY env or provided key) * -h, --host Webhook host type: 'local', 'docker', or custom hostname (default: local) * -w, --webhook Explicit webhook URL override * -v, --variables Template variables as JSON string (e.g. '{"name":"Alex"}') @@ -49,10 +49,6 @@ const projectRootDir = join(__dirname, '..'); dotenv.config({ path: join(projectRootDir, '.env') }); dotenv.config(); -// HTTP Connection Pooling for high concurrency load testing -const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 200 }); -const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 200 }); - // --- Argument Parsing --- const args = process.argv.slice(2); const config = { @@ -65,7 +61,7 @@ const config = { subject: 'Test Notification', message: 'This is a test notification from SimpleNS load test script.', baseUrl: process.env.BASE_URL || 'http://localhost:3000', - apiKey: process.env.NS_API_KEY || '4YCItWcuH2qJe3bXM9LbsbqefflWFlXlzvneMRSSQhU=', + apiKey: process.env.API_KEY || process.env.NS_API_KEY || '', hostType: 'local', webhookUrl: null, variables: null, @@ -178,6 +174,10 @@ if (!webhookUrl) { webhookUrl = `http://${host}:4000/webhook`; } +// HTTP Connection Pooling for high concurrency load testing +const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 200 }); +const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 200 }); + // Axios Client instance with connection pooling const apiClient = axios.create({ baseURL: config.baseUrl, @@ -387,4 +387,3 @@ main().catch(err => { console.error('Fatal execution error:', err); process.exit(1); }); - diff --git a/src/admin-alerts/admin-channel.interface.ts b/src/admin-alerts/admin-channel.interface.ts index 737d54f..99003c7 100644 --- a/src/admin-alerts/admin-channel.interface.ts +++ b/src/admin-alerts/admin-channel.interface.ts @@ -3,7 +3,12 @@ * Defines the contract for admin notification channel implementations */ -import type { AdminAlertType, AdminChannelType } from '@src/types/types.js'; +export type { + AdminChannelProvider, + AlertMetadata, + ChannelResult, + CredentialField, +} from '@src/types/types.js'; /** * Severity colors for Discord embeds @@ -28,72 +33,4 @@ export const ALERT_EMOJIS: Record = { /** * Result of a channel operation */ -export interface ChannelResult { - success: boolean; - error?: string; -} - -/** - * Metadata attached to alerts for rich formatting - */ -export interface AlertMetadata { - alertType?: AdminAlertType; - severity?: 'info' | 'warning' | 'critical'; - timestamp?: Date; - notificationId?: string; - channel?: string; - errorMessage?: string; -} - -/** - * Credential field definition for dynamic form generation - * Used by dashboard to render provider-specific configuration forms - */ -export interface CredentialField { - /** Field name (used as key in config object) */ - name: string; - /** Field type for input rendering */ - type: 'string' | 'url' | 'secret'; - /** Display label for the field */ - label: string; - /** Placeholder text for input */ - placeholder?: string; - /** Help text / description */ - description?: string; - /** Whether field is required */ - required: boolean; - /** Regex pattern for validation */ - pattern?: string; -} - -/** - * Interface for admin alert channel providers - * All channel implementations must implement this interface - */ -export interface AdminChannelProvider { - /** The channel type identifier */ - readonly channelType: AdminChannelType; - - /** Display name for UI */ - readonly displayName: string; - - /** - * Send an alert message through this channel - * @param message - The alert message to send - * @param metadata - Optional metadata for rich formatting - */ - send(message: string, metadata?: AlertMetadata): Promise; - - /** - * Test the channel connection - * @returns Result indicating if the connection test was successful - */ - testConnection(): Promise; - - /** - * Get credential schema for dynamic form generation - * @returns Array of credential field definitions - */ - getCredentialSchema(): CredentialField[]; -} diff --git a/src/admin-alerts/channel-registry.ts b/src/admin-alerts/channel-registry.ts index f46acd4..eba8ce8 100644 --- a/src/admin-alerts/channel-registry.ts +++ b/src/admin-alerts/channel-registry.ts @@ -3,8 +3,7 @@ * Extensible pattern - new channels register themselves here */ -import type { AdminChannelProvider, CredentialField } from './admin-channel.interface.js'; -import type { AdminChannelType } from '@src/types/types.js'; +import type { AdminChannelProvider, AdminChannelMeta, AdminChannelType } from '@src/types/types.js'; /** * Factory function type for creating channel provider instances @@ -67,12 +66,6 @@ export function getRegisteredChannelTypes(): AdminChannelType[] { /** * Admin channel metadata for dashboard */ -export interface AdminChannelMeta { - channelType: AdminChannelType; - displayName: string; - credentialFields: CredentialField[]; -} - /** * Get metadata for all registered admin channel providers * Used by dashboard for dynamic form generation diff --git a/src/api/controllers/admin-auth.controller.ts b/src/api/controllers/admin-auth.controller.ts new file mode 100644 index 0000000..f4dc32e --- /dev/null +++ b/src/api/controllers/admin-auth.controller.ts @@ -0,0 +1,202 @@ +/** + * Admin Authentication & Initial Setup Controller + * + * Manages first-time administrator account onboarding, authentication status checks, + * and credential verification using crypto.scrypt hashing and constant-time comparison. + */ + +import { Request, Response } from 'express'; +import { randomBytes, scryptSync, timingSafeEqual } from 'crypto'; +import system_config_model from '@src/database/models/system-config.models.js'; +import { adminSetupSchema, adminLoginSchema } from '@src/types/schemas.js'; +import type { AdminCredentialsDoc } from '@src/types/types.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; +import { generateAdminJwt } from '../utils/jwt.utils.js'; + +export const ADMIN_CREDENTIALS_KEY = 'admin_credentials'; +export const SESSION_COOKIE_NAME = 'simplens_session'; +const SESSION_COOKIE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days + +const setSessionCookie = (res: Response, token: string): void => { + res.cookie(SESSION_COOKIE_NAME, token, { + httpOnly: true, + secure: process.env.NODE_ENV === 'production', + sameSite: 'lax', + path: '/', + maxAge: SESSION_COOKIE_MAX_AGE_MS, + }); +}; + +/** + * Check if the administrator credentials have been configured + * GET /api/admin/auth/status + */ +export const getAuthStatus = async (_req: Request, res: Response): Promise => { + try { + const doc = await system_config_model.findOne({ key: ADMIN_CREDENTIALS_KEY }); + const isConfigured = Boolean(doc?.value); + + res.status(200).json({ isConfigured }); + } catch (err) { + logger.error('Failed to query admin auth status', err); + res.status(500).json({ error: 'Internal server error checking admin auth status' }); + } +}; + +/** + * Configure administrator account on initial setup + * POST /api/admin/auth/setup + */ +export const setupAdmin = async (req: Request, res: Response): Promise => { + try { + const validation = adminSetupSchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + error: 'Validation failed', + details: validation.error.flatten().fieldErrors, + }); + return; + } + + const { username, password } = validation.data; + + // Check if already configured + const existingDoc = await system_config_model.findOne({ key: ADMIN_CREDENTIALS_KEY }); + if (existingDoc?.value) { + res.status(400).json({ + error: 'Administrator credentials have already been configured. Initial setup is locked.', + }); + return; + } + + // Generate cryptographic salt and scrypt hash + const salt = randomBytes(32).toString('hex'); + const password_hash = scryptSync(password, salt, 64).toString('hex'); + + const credentialsData: AdminCredentialsDoc = { + username, + password_hash, + salt, + created_at: new Date().toISOString(), + }; + + await system_config_model.create({ + key: ADMIN_CREDENTIALS_KEY, + value: credentialsData, + }); + + logger.success(`Administrator account configured successfully for username: ${username}`); + + const user = { id: 'admin-1', username }; + const token = generateAdminJwt(user); + setSessionCookie(res, token); + + res.status(201).json({ + success: true, + message: 'Administrator account configured successfully.', + token, + user, + }); + } catch (err) { + logger.error('Error during administrator setup', err); + res.status(500).json({ error: 'Failed to configure administrator account' }); + } +}; + +/** + * Verify administrator credentials + * POST /api/admin/auth/verify + */ +export const verifyAdmin = async (req: Request, res: Response): Promise => { + try { + const validation = adminLoginSchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + isValid: false, + message: 'Username and password are required', + }); + return; + } + + const { username, password } = validation.data; + + const doc = await system_config_model.findOne({ key: ADMIN_CREDENTIALS_KEY }); + + if (doc?.value) { + const credentials = doc.value as AdminCredentialsDoc; + + // Username match check + if (credentials.username !== username) { + res.status(401).json({ isValid: false, message: 'Invalid username or password' }); + return; + } + + // Compute scrypt hash with stored salt + const computedHash = scryptSync(password, credentials.salt, 64).toString('hex'); + const storedHashBuf = Buffer.from(credentials.password_hash, 'hex'); + const computedHashBuf = Buffer.from(computedHash, 'hex'); + + if (storedHashBuf.length === computedHashBuf.length && timingSafeEqual(storedHashBuf, computedHashBuf)) { + const user = { id: 'admin-1', username: credentials.username }; + const token = generateAdminJwt(user); + setSessionCookie(res, token); + res.status(200).json({ + success: true, + isValid: true, + token, + user, + }); + return; + } + + res.status(401).json({ isValid: false, message: 'Invalid username or password' }); + return; + } + + // Fallback check against process.env if present + const envUser = process.env.ADMIN_USERNAME; + const envPass = process.env.ADMIN_PASSWORD; + if (envUser && envPass && username === envUser && password === envPass) { + const user = { id: 'admin-1', username: envUser }; + const token = generateAdminJwt(user); + setSessionCookie(res, token); + res.status(200).json({ + success: true, + isValid: true, + token, + user, + }); + return; + } + + res.status(401).json({ isValid: false, message: 'Invalid username or password' }); + } catch (err) { + logger.error('Error verifying administrator credentials', err); + res.status(500).json({ error: 'Failed to verify administrator credentials' }); + } +}; + +/** + * Log out administrator and clear session cookie + * POST /api/admin/auth/logout + */ +export const logoutAdmin = async (_req: Request, res: Response): Promise => { + try { + res.clearCookie(SESSION_COOKIE_NAME, { + path: '/', + httpOnly: true, + secure: process.env.NODE_ENV === 'production', + sameSite: 'lax', + }); + res.status(200).json({ + success: true, + message: 'Logged out successfully.', + }); + } catch (err) { + logger.error('Error during logout', err); + res.status(500).json({ error: 'Failed to log out' }); + } +}; + +export const signupAdmin = setupAdmin; +export const loginAdmin = verifyAdmin; diff --git a/src/api/controllers/api-key.controller.ts b/src/api/controllers/api-key.controller.ts new file mode 100644 index 0000000..33360ad --- /dev/null +++ b/src/api/controllers/api-key.controller.ts @@ -0,0 +1,199 @@ +/** + * API Key Management Controller + * + * Handles creation, listing, revocation, deletion, and usage aggregation + * for external service API keys. + */ + +import type { Request, Response } from 'express'; +import crypto from 'crypto'; +import api_key_model from '@src/database/models/api-key.models.js'; +import notification_model from '@src/database/models/notification.models.js'; +import { createApiKeySchema } from '@src/types/schemas.js'; +import type { ApiKeyDoc, ApiKeyResponse } from '@src/types/types.js'; +import { ApiKeyCache } from '../utils/api-key-cache.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +/** + * Format an API Key document for API response (strips secret hash) + */ +function toApiKeyResponse(doc: ApiKeyDoc): ApiKeyResponse { + return { + key_id: doc.key_id, + name: doc.name, + type: doc.type || 'notification', + key_prefix: doc.key_prefix, + status: doc.status, + expires_at: doc.expires_at || null, + usage: { + total_requests: doc.usage?.total_requests || 0, + total_notifications: doc.usage?.total_notifications || 0, + by_channel: doc.usage?.by_channel instanceof Map + ? Object.fromEntries(doc.usage.by_channel) + : (doc.usage?.by_channel || {}), + last_used_at: doc.usage?.last_used_at || null, + }, + created_at: doc.created_at || new Date(), + }; +} + +/** + * GET /api/keys - List all API keys + */ +export const listApiKeys = async (_req: Request, res: Response): Promise => { + try { + const keys = await api_key_model.find().sort({ created_at: -1 }); + const responseKeys = keys.map((k) => toApiKeyResponse(k.toObject() as ApiKeyDoc)); + res.status(200).json({ keys: responseKeys }); + } catch (err) { + logger.error('Failed to list API keys', err); + res.status(500).json({ error: 'Failed to retrieve API keys' }); + } +}; + +/** + * POST /api/keys - Create a new API key + */ +export const createApiKey = async (req: Request, res: Response): Promise => { + try { + const validation = createApiKeySchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + error: 'Validation failed', + details: validation.error.flatten().fieldErrors, + }); + return; + } + + const { name, type, expires_at } = validation.data; + + // Generate cryptographically random API key + const rawKey = `sns_live_${crypto.randomBytes(24).toString('base64url')}`; + const key_prefix = `${rawKey.slice(0, 16)}...`; + const key_hash = crypto.createHash('sha256').update(rawKey).digest('hex'); + const key_id = `ak_${crypto.randomUUID()}`; + + const newDoc = await api_key_model.create({ + key_id, + name, + type: type || 'notification', + key_prefix, + key_hash, + status: 'active', + expires_at: expires_at || null, + usage: { + total_requests: 0, + total_notifications: 0, + by_channel: {}, + last_used_at: null, + }, + }); + + logger.success(`Created new API key: ${name} (${key_id})`); + + res.status(201).json({ + key: toApiKeyResponse(newDoc.toObject() as ApiKeyDoc), + raw_key: rawKey, + }); + } catch (err) { + logger.error('Failed to create API key', err); + res.status(500).json({ error: 'Failed to create API key' }); + } +}; + +/** + * GET /api/keys/:id - Get single API key details & notification status breakdown + */ +export const getApiKeyUsage = async (req: Request, res: Response): Promise => { + try { + const { id } = req.params; + const key = await api_key_model.findOne({ key_id: id }); + + if (!key) { + res.status(404).json({ error: 'API key not found' }); + return; + } + + // Aggregate delivery outcomes from notifications collection + const statusBreakdown = await notification_model.aggregate([ + { $match: { api_key_id: id } }, + { $group: { _id: '$status', count: { $sum: 1 } } }, + ]); + + const statusCounts: Record = { + pending: 0, + processing: 0, + delivered: 0, + failed: 0, + }; + + for (const item of statusBreakdown) { + if (item._id && typeof item.count === 'number') { + statusCounts[item._id] = item.count; + } + } + + res.status(200).json({ + key: toApiKeyResponse(key.toObject() as ApiKeyDoc), + status_breakdown: statusCounts, + }); + } catch (err) { + logger.error(`Failed to get API key usage for ${req.params.id}`, err); + res.status(500).json({ error: 'Failed to retrieve API key usage' }); + } +}; + +/** + * POST /api/keys/:id/revoke - Revoke an API key + */ +export const revokeApiKey = async (req: Request, res: Response): Promise => { + try { + const { id } = req.params; + const key = await api_key_model.findOneAndUpdate( + { key_id: id }, + { $set: { status: 'revoked' } }, + { new: true } + ); + + if (!key) { + res.status(404).json({ error: 'API key not found' }); + return; + } + + logger.info(`Revoked API key: ${key.name} (${id})`); + ApiKeyCache.invalidate(id); + res.status(200).json({ + success: true, + message: 'API key has been revoked successfully', + key: toApiKeyResponse(key.toObject() as ApiKeyDoc), + }); + } catch (err) { + logger.error(`Failed to revoke API key ${req.params.id}`, err); + res.status(500).json({ error: 'Failed to revoke API key' }); + } +}; + +/** + * DELETE /api/keys/:id - Permanently delete an API key + */ +export const deleteApiKey = async (req: Request, res: Response): Promise => { + try { + const { id } = req.params; + const result = await api_key_model.deleteOne({ key_id: id }); + + if (result.deletedCount === 0) { + res.status(404).json({ error: 'API key not found' }); + return; + } + + logger.info(`Deleted API key (${id})`); + ApiKeyCache.invalidate(id); + res.status(200).json({ + success: true, + message: 'API key deleted successfully', + }); + } catch (err) { + logger.error(`Failed to delete API key ${req.params.id}`, err); + res.status(500).json({ error: 'Failed to delete API key' }); + } +}; diff --git a/src/api/controllers/channel-routing.controller.ts b/src/api/controllers/channel-routing.controller.ts new file mode 100644 index 0000000..f625f43 --- /dev/null +++ b/src/api/controllers/channel-routing.controller.ts @@ -0,0 +1,125 @@ +/** + * Channel Routing Controller + * + * Manages channel default and fallback provider routing and Kafka partition scaling + */ + +import { Request, Response } from 'express'; +import { ChannelRoutingService } from '@src/plugins/index.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +/** + * GET /api/channels/routing + * List all channel routings + */ +export const listChannelRoutings = async (_req: Request, res: Response): Promise => { + try { + const routings = await ChannelRoutingService.listChannelRoutings(); + res.json({ routings }); + } catch (err) { + logger.error('Error listing channel routings:', err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to list channel routings' + }); + } +}; + +/** + * GET /api/channels/routing/:channel + * Get routing for a specific channel + */ +export const getChannelRouting = async (req: Request, res: Response): Promise => { + try { + const routing = await ChannelRoutingService.getChannelRouting(req.params.channel); + if (!routing) { + res.status(404).json({ + error: 'Not Found', + message: `Routing for channel '${req.params.channel}' not found` + }); + return; + } + + res.json({ routing }); + } catch (err) { + logger.error(`Error getting channel routing for '${req.params.channel}':`, err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to retrieve channel routing' + }); + } +}; + +/** + * PUT /api/channels/routing/:channel + * Set or update routing for a channel + */ +export const setChannelRouting = async (req: Request, res: Response): Promise => { + try { + const channel = req.params.channel.trim(); + const { default_provider_id, fallback_provider_ids, partitions } = req.body; + + if (!default_provider_id || typeof default_provider_id !== 'string') { + res.status(400).json({ + error: 'Bad Request', + message: 'default_provider_id is required' + }); + return; + } + + if (fallback_provider_ids !== undefined && !Array.isArray(fallback_provider_ids)) { + res.status(400).json({ + error: 'Bad Request', + message: 'fallback_provider_ids must be an array of provider IDs' + }); + return; + } + + if (partitions !== undefined && (typeof partitions !== 'number' || partitions < 1)) { + res.status(400).json({ + error: 'Bad Request', + message: 'partitions must be a positive integer' + }); + return; + } + + const routing = await ChannelRoutingService.setChannelRouting(channel, { + default_provider_id, + fallback_provider_ids, + partitions, + }); + + res.json({ + message: `Routing for channel '${channel}' updated successfully`, + routing + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error updating channel routing for '${req.params.channel}': ${message}`); + const statusCode = message.includes('Cannot decrease partitions') || message.includes('not found') ? 400 : 500; + res.status(statusCode).json({ + error: 'Failed to set channel routing', + message + }); + } +}; + +/** + * DELETE /api/channels/routing/:channel + * Delete routing for a channel + */ +export const deleteChannelRouting = async (req: Request, res: Response): Promise => { + try { + await ChannelRoutingService.deleteChannelRouting(req.params.channel); + res.json({ + message: `Routing for channel '${req.params.channel}' deleted successfully` + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error deleting channel routing for '${req.params.channel}': ${message}`); + res.status(500).json({ + error: 'Failed to delete channel routing', + message + }); + } +}; diff --git a/src/api/controllers/health.controller.ts b/src/api/controllers/health.controller.ts new file mode 100644 index 0000000..bd37974 --- /dev/null +++ b/src/api/controllers/health.controller.ts @@ -0,0 +1,68 @@ +import type { Request, Response } from 'express'; +import mongoose from 'mongoose'; +import { getRedisClient } from '@src/config/redis.config.js'; +import { runHealthChecks } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeCheck, ApiHealthResponse } from '@src/types/types.js'; + +let isAppShuttingDown = false; + +export const setAppShuttingDown = (value: boolean): void => { + isAppShuttingDown = value; +}; + +export const getApiReadinessChecks = (): HealthProbeCheck[] => [ + { + name: 'mongodb', + check: () => !isAppShuttingDown && mongoose.connection.readyState === 1 + }, + { + name: 'redis', + check: () => { + if (isAppShuttingDown) return false; + try { + return getRedisClient().status === 'ready'; + } catch { + return false; + } + } + } +]; + +export const getApiLivenessChecks = (): HealthProbeCheck[] => [ + { + name: 'process', + check: () => !isAppShuttingDown + } +]; + +export const getHealth = async (req: Request, res: Response): Promise => { + const { ok, results } = await runHealthChecks(getApiReadinessChecks()); + const payload: ApiHealthResponse = { + status: ok ? 'healthy' : 'unhealthy', + timestamp: new Date().toISOString(), + checks: results + }; + res.status(ok ? 200 : 503).json(payload); +}; + +export const getLiveness = async (req: Request, res: Response): Promise => { + const { ok, results } = await runHealthChecks(getApiLivenessChecks()); + const payload: ApiHealthResponse = { + service: 'api', + status: ok ? 'healthy' : 'unhealthy', + timestamp: new Date().toISOString(), + checks: results + }; + res.status(ok ? 200 : 503).json(payload); +}; + +export const getReadiness = async (req: Request, res: Response): Promise => { + const { ok, results } = await runHealthChecks(getApiReadinessChecks()); + const payload: ApiHealthResponse = { + service: 'api', + status: ok ? 'ready' : 'not_ready', + timestamp: new Date().toISOString(), + checks: results + }; + res.status(ok ? 200 : 503).json(payload); +}; diff --git a/src/api/controllers/notification.controller.ts b/src/api/controllers/notification.controller.ts index 1302b3e..1237fd2 100644 --- a/src/api/controllers/notification.controller.ts +++ b/src/api/controllers/notification.controller.ts @@ -1,8 +1,16 @@ import type {Request, Response} from 'express'; import { safeValidateNotificationRequest, safeValidateBatchNotificationRequest } from '@src/types/schemas.js'; -import { convert_notification_request_to_notification_schema, convert_batch_notification_schema_to_notification_schema, DuplicateNotificationError, InvalidProviderChannelError, InvalidContentSchemaError, ProviderNotFoundError} from '../utils/utils.js'; +import { + convert_notification_request_to_notification_schema, + convert_batch_notification_schema_to_notification_schema, + DuplicateNotificationError, + InvalidProviderChannelError, + InvalidContentSchemaError, + ProviderNotFoundError, + process_notifications, + updateApiKeyNotificationUsage, +} from '../utils/utils.js'; import { notification } from '@src/types/types.js'; -import { process_notifications } from '@src/api/utils/utils.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; export const notification_controller = async (req: Request, res: Response)=>{ @@ -11,8 +19,12 @@ export const notification_controller = async (req: Request, res: Response)=>{ const result = safeValidateNotificationRequest(data); if(result.success){ const request_data = result.data; - const notifications: notification[] = await convert_notification_request_to_notification_schema(request_data); + const apiKeyId = req.apiKey?.key_id; + const notifications: notification[] = await convert_notification_request_to_notification_schema(request_data, apiKeyId); const response = await process_notifications(notifications); + + updateApiKeyNotificationUsage(req.apiKey, notifications, response?.created_count ?? 0); + res.status(202).json({ message: 'Notifications are being processed', ...response}); } else{ @@ -66,8 +78,12 @@ export const batch_notification_controller = async (req: Request, res: Response) const result = safeValidateBatchNotificationRequest(data); if(result.success){ const request_data = result.data; - const notifications: notification[] = await convert_batch_notification_schema_to_notification_schema(request_data); + const apiKeyId = req.apiKey?.key_id; + const notifications: notification[] = await convert_batch_notification_schema_to_notification_schema(request_data, apiKeyId); const response = await process_notifications(notifications); + + updateApiKeyNotificationUsage(req.apiKey, notifications, response?.created_count ?? 0); + res.status(202).json({ message: 'Notifications are being processed', ...response}); } else{ diff --git a/src/api/controllers/notifications-management.controller.ts b/src/api/controllers/notifications-management.controller.ts index 5f7b030..61e61c5 100644 --- a/src/api/controllers/notifications-management.controller.ts +++ b/src/api/controllers/notifications-management.controller.ts @@ -104,6 +104,7 @@ export const listNotifications = async (req: Request, res: Response): Promise => { - try { - if (!PluginRegistry.isInitialized()) { - res.status(503).json({ - error: 'Plugin system not initialized', - message: 'The plugin system has not been initialized yet. Ensure providers are loaded.' - }); - return; - } - - const metadata = PluginRegistry.getPluginMetadata(); - res.json(metadata); - } catch (err) { - logger.error('Error getting plugin metadata:', err); - res.status(500).json({ - error: 'Internal server error', - message: 'Failed to retrieve plugin metadata' - }); + try { + if (!PluginRegistry.isInitialized()) { + res.status(503).json({ + error: 'Plugin system not initialized', + message: 'The plugin system has not been initialized yet. Ensure providers are loaded.' + }); + return; + } + + const metadata = PluginRegistry.getPluginMetadata(); + res.json(metadata); + } catch (err) { + logger.error('Error getting plugin metadata:', err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to retrieve plugin metadata' + }); + } +}; + +/** + * GET /api/plugins/installed + * List all installed npm plugins in MongoDB + */ +export const listInstalledPlugins = async (_req: Request, res: Response): Promise => { + try { + const plugins = await PluginManagerService.listInstalledPlugins(); + res.json({ plugins }); + } catch (err) { + logger.error('Error listing installed plugins:', err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to list installed plugins' + }); + } +}; + +/** + * GET /api/plugins/catalog/:category + * Proxy the public plugin catalog server-side to avoid browser CORS restrictions. + */ +export const listPluginCatalog = async (req: Request, res: Response): Promise => { + const category = req.params.category; + if (category !== 'official' && category !== 'community') { + res.status(400).json({ + error: 'Bad Request', + message: 'Catalog category must be official or community', + }); + return; + } + + try { + const response = await axios.get( + `${PLUGIN_CATALOG_BASE_URL}/${category}`, + { timeout: 10000 } + ); + const catalog = Array.isArray(response.data) ? response.data : []; + res.json(catalog); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error loading ${category} plugin catalog: ${message}`); + res.status(502).json({ + error: 'Bad Gateway', + message: `Failed to load ${category} plugin catalog`, + }); + } +}; + +/** + * POST /api/plugins/install + * Install an npm plugin package + */ +export const installPlugin = async (req: Request, res: Response): Promise => { + try { + const validation = installPluginPayloadSchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + error: 'Bad Request', + message: 'Invalid plugin installation payload', + details: validation.error.flatten().fieldErrors, + }); + return; } + + const { package: packageName, version, auth } = validation.data; + const plugin = await PluginManagerService.installPlugin(packageName.trim(), version?.trim(), auth); + res.status(201).json({ + message: `Plugin '${packageName}' installed successfully`, + plugin + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error installing plugin: ${message}`); + + const isBadRequest = + message.includes('not a valid SimpleNS plugin') || + message.includes('invalid plugin manifest') || + message.includes('npm authentication failed') || + message.includes('not found') || + message.includes('Missing required provider methods'); + + res.status(isBadRequest ? 400 : 500).json({ + error: isBadRequest ? 'Bad Request' : 'Failed to install plugin', + message + }); + } +}; + +/** + * PUT /api/plugins/version + * Upgrade or downgrade an installed plugin + */ +export const changePluginVersion = async (req: Request, res: Response): Promise => { + try { + const validation = changePluginVersionSchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + error: 'Bad Request', + message: 'Invalid plugin version change payload', + details: validation.error.flatten().fieldErrors, + }); + return; + } + + const { package: packageName, version } = validation.data; + + const plugin = await PluginManagerService.changePluginVersion(packageName.trim(), version.trim()); + res.json({ + message: `Plugin '${packageName}' changed to version '${version}'`, + plugin + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error changing plugin version: ${message}`); + res.status(500).json({ + error: 'Failed to change plugin version', + message + }); + } }; + +/** + * DELETE /api/plugins/:package + * Uninstall an installed plugin + */ +export const uninstallPlugin = async (req: Request, res: Response): Promise => { + try { + let rawPackageName = (req.query.package as string) || (req.body?.package as string); + + if (!rawPackageName) { + if (req.params.scope && req.params.package) { + rawPackageName = `${req.params.scope}/${req.params.package}`; + } else if (req.params.package) { + rawPackageName = decodeURIComponent(req.params.package); + } + } + + const validation = uninstallPluginSchema.safeParse({ package: rawPackageName }); + if (!validation.success) { + res.status(400).json({ + error: 'Bad Request', + message: 'Invalid npm package name format', + details: validation.error.flatten().fieldErrors, + }); + return; + } + + const { package: packageName } = validation.data; + + await PluginManagerService.uninstallPlugin(packageName.trim()); + res.json({ + message: `Plugin '${packageName.trim()}' uninstalled successfully` + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error uninstalling plugin: ${message}`); + const statusCode = message.includes('depend on it') ? 400 : 500; + res.status(statusCode).json({ + error: 'Failed to uninstall plugin', + message + }); + } +}; + +/** + * GET /api/plugins/npm-auth + * Retrieve status of configured npm auth (with masked token) + */ +export const getNpmAuth = async (_req: Request, res: Response): Promise => { + try { + const status = await NpmAuthService.getAuthStatus(); + res.json(status); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error retrieving npm auth status: ${message}`); + res.status(500).json({ + error: 'Failed to retrieve npm auth status', + message, + }); + } +}; + +/** + * POST /api/plugins/npm-auth + * Configure/save npm auth token and registry URL + */ +export const saveNpmAuth = async (req: Request, res: Response): Promise => { + try { + const validation = npmAuthConfigSchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + error: 'Bad Request', + message: 'Invalid npm auth configuration', + details: validation.error.flatten().fieldErrors, + }); + return; + } + + const { token, registry_url, scope } = validation.data; + const status = await NpmAuthService.saveNpmAuth(token, registry_url, scope); + res.status(200).json({ + message: 'npm auth configuration saved successfully', + status, + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error saving npm auth configuration: ${message}`); + res.status(500).json({ + error: 'Failed to save npm auth configuration', + message, + }); + } +}; + +/** + * DELETE /api/plugins/npm-auth + * Delete saved npm auth configuration (specific registry or all) + */ +export const deleteNpmAuth = async (req: Request, res: Response): Promise => { + try { + const target = (req.query.id as string) || (req.query.scope as string) || (req.body?.id as string); + await NpmAuthService.deleteNpmAuth(target); + res.json({ + message: target + ? `npm auth configuration for '${target}' removed successfully` + : 'npm auth configuration removed successfully', + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error deleting npm auth configuration: ${message}`); + res.status(500).json({ + error: 'Failed to delete npm auth configuration', + message, + }); + } +}; + diff --git a/src/api/controllers/providers.controller.ts b/src/api/controllers/providers.controller.ts new file mode 100644 index 0000000..7220143 --- /dev/null +++ b/src/api/controllers/providers.controller.ts @@ -0,0 +1,257 @@ +/** + * Providers Controller + * + * Manages provider configurations, credential testing, and lifecycle + */ + +import { Request, Response } from 'express'; +import { ProviderManagerService } from '@src/plugins/index.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +/** + * GET /api/providers + * List all configured providers + */ +export const listProviders = async (_req: Request, res: Response): Promise => { + try { + const providers = await ProviderManagerService.listProviders(); + res.json({ providers }); + } catch (err) { + logger.error('Error listing providers:', err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to list providers' + }); + } +}; + +/** + * GET /api/providers/:id + * Get single provider by ID + */ +export const getProvider = async (req: Request, res: Response): Promise => { + try { + const includeDecrypted = req.query.include_decrypted === 'true'; + const provider = await ProviderManagerService.getProvider(req.params.id, includeDecrypted); + + if (!provider) { + res.status(404).json({ + error: 'Not Found', + message: `Provider '${req.params.id}' not found` + }); + return; + } + + res.json({ provider }); + } catch (err) { + logger.error(`Error getting provider '${req.params.id}':`, err); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to retrieve provider' + }); + } +}; + +/** + * POST /api/providers + * Create a new provider instance + */ +export const createProvider = async (req: Request, res: Response): Promise => { + try { + const { id, plugin_name, credentials, options, enabled } = req.body; + + if (!id || typeof id !== 'string') { + res.status(400).json({ + error: 'Bad Request', + message: 'Provider id is required and must be a string' + }); + return; + } + + if (!plugin_name || typeof plugin_name !== 'string') { + res.status(400).json({ + error: 'Bad Request', + message: 'plugin_name is required' + }); + return; + } + + if (!credentials || typeof credentials !== 'object') { + res.status(400).json({ + error: 'Bad Request', + message: 'credentials object is required' + }); + return; + } + + const provider = await ProviderManagerService.createProvider({ + id: id.trim(), + plugin_name: plugin_name.trim(), + credentials, + options: options || {}, + enabled: enabled !== false, + }); + + res.status(201).json({ + message: `Provider '${id}' created successfully`, + provider + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error creating provider: ${message}`); + const statusCode = message.includes('already exists') ? 409 : message.includes('not installed') ? 404 : 500; + res.status(statusCode).json({ + error: 'Failed to create provider', + message + }); + } +}; + +/** + * PUT /api/providers/:id + * Update an existing provider instance + */ +export const updateProvider = async (req: Request, res: Response): Promise => { + try { + const { credentials, options, enabled } = req.body; + + const provider = await ProviderManagerService.updateProvider(req.params.id, { + credentials, + options, + enabled: typeof enabled === 'boolean' ? enabled : undefined, + }); + + res.json({ + message: `Provider '${req.params.id}' updated successfully`, + provider + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error updating provider '${req.params.id}': ${message}`); + const statusCode = message.includes('not found') ? 404 : 500; + res.status(statusCode).json({ + error: 'Failed to update provider', + message + }); + } +}; + +/** + * DELETE /api/providers/:id + * Delete a provider instance + */ +export const deleteProvider = async (req: Request, res: Response): Promise => { + try { + await ProviderManagerService.deleteProvider(req.params.id); + res.json({ + message: `Provider '${req.params.id}' deleted successfully` + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error deleting provider '${req.params.id}': ${message}`); + const statusCode = message.includes('configured in channel') ? 400 : message.includes('not found') ? 404 : 500; + res.status(statusCode).json({ + error: 'Failed to delete provider', + message + }); + } +}; + +/** + * POST /api/providers/test + * Test provider connection / credentials + */ +export const testProvider = async (req: Request, res: Response): Promise => { + try { + const { provider_id, plugin_name, credentials, options } = req.body; + + const result = await ProviderManagerService.testProviderConnection({ + provider_id, + plugin_name, + credentials, + options, + }); + + if (result.success) { + res.json(result); + } else { + res.status(400).json(result); + } + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error testing provider: ${message}`); + res.status(500).json({ + success: false, + message + }); + } +}; + +/** + * GET /api/providers/rate-limits + * Get real-time rate limit status across all configured providers + */ +export const getProvidersRateLimits = async (_req: Request, res: Response): Promise => { + try { + const { getAllProvidersRateLimitStatus } = await import('@src/processors/shared/rate-limiter.js'); + const result = await getAllProvidersRateLimitStatus(); + res.json(result); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error fetching providers rate limits: ${message}`); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to retrieve providers rate limits' + }); + } +}; + +/** + * GET /api/providers/:id/rate-limit + * Get real-time rate limit status for a specific provider + */ +export const getProviderRateLimit = async (req: Request, res: Response): Promise => { + try { + const { getProviderRateLimitStatus } = await import('@src/processors/shared/rate-limiter.js'); + const rateLimitStatus = await getProviderRateLimitStatus(req.params.id); + + if (!rateLimitStatus) { + res.status(404).json({ + error: 'Not Found', + message: `Provider '${req.params.id}' rate limit status not found` + }); + return; + } + + res.json({ rate_limit: rateLimitStatus }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error fetching provider '${req.params.id}' rate limit: ${message}`); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to retrieve provider rate limit' + }); + } +}; + +/** + * POST /api/providers/:id/rate-limit/reset + * Manually reset rate limiter tokens and telemetry for a provider + */ +export const resetProviderRateLimit = async (req: Request, res: Response): Promise => { + try { + const { resetRateLimiter } = await import('@src/processors/shared/rate-limiter.js'); + await resetRateLimiter(req.params.id); + res.json({ + message: `Rate limit for provider '${req.params.id}' has been reset successfully.` + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.error(`Error resetting provider '${req.params.id}' rate limit: ${message}`); + res.status(500).json({ + error: 'Internal server error', + message: 'Failed to reset provider rate limit' + }); + } +}; + diff --git a/src/api/controllers/settings.controller.ts b/src/api/controllers/settings.controller.ts new file mode 100644 index 0000000..80d7786 --- /dev/null +++ b/src/api/controllers/settings.controller.ts @@ -0,0 +1,76 @@ +/** + * Dynamic Operational Settings Controller + * + * Handles reading, updating, and resetting operational configuration settings. + */ + +import { Request, Response } from 'express'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; +import { partialOperationalSettingsSchema } from '@src/types/schemas.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +/** + * Get current operational settings + * GET /api/settings + */ +export const getSettings = async (_req: Request, res: Response): Promise => { + try { + const settings = dynamicConfig.getAll(); + res.status(200).json({ + success: true, + settings, + }); + } catch (err) { + logger.error('Failed to retrieve operational settings', err); + res.status(500).json({ error: 'Failed to retrieve operational settings' }); + } +}; + +/** + * Update operational settings + * PUT /api/settings + */ +export const updateSettings = async (req: Request, res: Response): Promise => { + try { + const validation = partialOperationalSettingsSchema.safeParse(req.body); + if (!validation.success) { + res.status(400).json({ + error: 'Validation failed', + details: validation.error.flatten().fieldErrors, + }); + return; + } + + const updatedSettings = await dynamicConfig.updateSettings(validation.data); + logger.success('Operational settings updated and broadcasted'); + + res.status(200).json({ + success: true, + message: 'Operational settings updated and broadcasted successfully', + settings: updatedSettings, + }); + } catch (err) { + logger.error('Failed to update operational settings', err); + res.status(500).json({ error: 'Failed to update operational settings' }); + } +}; + +/** + * Reset operational settings to system defaults + * POST /api/settings/reset + */ +export const resetSettings = async (_req: Request, res: Response): Promise => { + try { + const reset = await dynamicConfig.resetSettings(); + logger.info('Operational settings reset to system defaults and broadcasted'); + + res.status(200).json({ + success: true, + message: 'Operational settings reset to system defaults successfully', + settings: reset, + }); + } catch (err) { + logger.error('Failed to reset operational settings', err); + res.status(500).json({ error: 'Failed to reset operational settings' }); + } +}; diff --git a/src/api/controllers/webhook.controller.ts b/src/api/controllers/webhook.controller.ts new file mode 100644 index 0000000..e3977e9 --- /dev/null +++ b/src/api/controllers/webhook.controller.ts @@ -0,0 +1,27 @@ +/** + * Webhook Controller + * + * Handles incoming notification status callbacks (e.g. delivery receipts, provider webhooks). + */ + +import type { Request, Response } from 'express'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +export const handleWebhookCallback = async (req: Request, res: Response): Promise => { + try { + const payload = req.body; + + logger.info('[Webhook] Received callback:', { + request_id: payload?.request_id, + notification_id: payload?.notification_id, + status: payload?.status, + channel: payload?.channel, + occurred_at: payload?.occurred_at, + }); + + res.status(200).json({ received: true }); + } catch (error) { + logger.error('[Webhook] Error processing callback:', error); + res.status(400).json({ received: false, error: 'Invalid payload' }); + } +}; diff --git a/src/api/middlewares/auth_middleware.ts b/src/api/middlewares/auth_middleware.ts index ba31f95..bc5b02e 100644 --- a/src/api/middlewares/auth_middleware.ts +++ b/src/api/middlewares/auth_middleware.ts @@ -1,37 +1,147 @@ +/** + * Core Authentication Middleware + * + * Authenticates API requests using: + * 1. Admin JWT tokens (issued for the Next.js Dashboard). + * 2. Dynamic API Keys (prefixed with 'sns_' and stored in MongoDB). + */ + import type { Request, Response, NextFunction } from 'express'; -import { env } from '@src/config/env.config.js'; import crypto from 'crypto'; +import api_key_model from '@src/database/models/api-key.models.js'; +import { verifyAdminJwt } from '../utils/jwt.utils.js'; +import { ApiKeyCache, ApiKeyUsageAggregator } from '../utils/api-key-cache.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; +import type { ApiKeyDoc, AdminJwtPayload } from '@src/types/types.js'; + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace Express { + interface Request { + user?: AdminJwtPayload; + apiKey?: ApiKeyDoc; + } + } +} -export const auth_middleware = (req: Request, res: Response, next: NextFunction) => { +async function handleDynamicApiKey( + token: string, + req: Request, + res: Response, + next: NextFunction +): Promise { try { - const authHeader = req.headers.authorization; - if (!authHeader || !authHeader.startsWith('Bearer ')) { - return res.status(401).json({ message: "API KEY missing in Authorization Header or 'Bearer' is missing" }); + const tokenHash = crypto.createHash('sha256').update(token).digest('hex'); + + let keyDoc = ApiKeyCache.get(tokenHash); + if (!keyDoc) { + const dbDoc = await api_key_model.findOne({ key_hash: tokenHash, status: 'active' }); + if (!dbDoc) { + res.status(401).json({ message: 'Invalid API KEY' }); + return; + } + const rawExpiresAt = (dbDoc as unknown as { expires_at?: Date | string | null }).expires_at; + keyDoc = (typeof dbDoc.toObject === 'function' ? dbDoc.toObject() : dbDoc) as ApiKeyDoc; + if (rawExpiresAt && !keyDoc.expires_at) { + keyDoc.expires_at = typeof rawExpiresAt === 'string' ? new Date(rawExpiresAt) : rawExpiresAt; + } + ApiKeyCache.set(tokenHash, keyDoc); } - const api_key = authHeader.split(' ')[1]; - if (!env.NS_API_KEY || typeof env.NS_API_KEY !== 'string' || env.NS_API_KEY.length === 0) { - logger.error('NS_API_KEY is not configured in the environment.'); - return res.status(500).json({ message: 'Server configuration error' }); + if (keyDoc.expires_at && new Date() > new Date(keyDoc.expires_at)) { + ApiKeyCache.invalidate(tokenHash); + res.status(401).json({ message: 'API key has expired' }); + return; } - if (api_key && typeof api_key === 'string') { - const apiKeyBuf = Buffer.from(api_key); - const serverKeyBuf = Buffer.from(env.NS_API_KEY); + const fullPath = (req.originalUrl || `${req.baseUrl || ''}${req.path}`).split('?')[0]; + const isAdminCredentialRoute = /(?:\/api)?(?:\/admin)?\/auth\/(?:setup|signup|login|verify)/.test(fullPath); + if (isAdminCredentialRoute) { + res.status(403).json({ + error: 'Forbidden: API keys cannot access administrator credential setup or verification routes.', + }); + return; + } + + const keyType = keyDoc.type || 'notification'; + const isManagementRoute = + fullPath.includes('/api/providers') || + fullPath.includes('/api/plugins') || + fullPath.includes('/api/channels/routing') || + fullPath.includes('/api/admin-channels') || + fullPath.includes('/api/settings') || + fullPath.includes('/api/keys') || + fullPath.includes('/api/dashboard') || + fullPath.includes('/api/alerts') || + (fullPath.includes('/api/templates') && req.method !== 'GET'); + + if (isManagementRoute && keyType !== 'admin') { + res.status(403).json({ + error: 'Forbidden: API key does not have management permissions. Requires admin key.', + }); + return; + } + + req.apiKey = keyDoc; + + ApiKeyUsageAggregator.recordRequest(keyDoc.key_id); - // Only use timingSafeEqual if lengths are equal. - let isValid = false; - if (apiKeyBuf.length === serverKeyBuf.length) { - isValid = crypto.timingSafeEqual(apiKeyBuf, serverKeyBuf); + next(); + } catch (err) { + logger.error('Error verifying dynamic API key', err); + res.status(500).json({ message: 'Internal Server Error' }); + } +} + +export const auth_middleware = ( + req: Request, + res: Response, + next: NextFunction +): void | Promise => { + try { + const authHeader = req.headers.authorization; + const xApiKeyHeader = req.headers['x-api-key']; + + let token: string | undefined; + + if (authHeader) { + if (!authHeader.startsWith('Bearer ')) { + res.status(401).json({ message: "API KEY missing in Authorization Header or 'Bearer' is missing" }); + return; } - if (isValid) { - return next(); + token = authHeader.slice(7).trim(); + } else if (typeof xApiKeyHeader === 'string' && xApiKeyHeader.trim().length > 0) { + token = xApiKeyHeader.trim(); + } else if (req.headers.cookie) { + const match = req.headers.cookie.match(/(?:^|;\s*)simplens_session=([^;]+)/); + if (match) { + token = decodeURIComponent(match[1]).trim(); } } - return res.status(401).json({ message: "Invalid API KEY" }); + + if (!token) { + res.status(401).json({ message: "API KEY missing in Authorization Header or 'Bearer' is missing" }); + return; + } + + // 1. Dynamic API Key (created via Dashboard / API) + if (token.startsWith('sns_')) { + return handleDynamicApiKey(token, req, res, next); + } + + // 2. Admin JWT (issued for Dashboard session) + const adminPayload = verifyAdminJwt(token); + if (adminPayload) { + req.user = adminPayload; + return next(); + } + + // 3. Invalid token + res.status(401).json({ message: 'Invalid API KEY' }); + return; } catch (err: unknown) { - logger.error(`Error in auth middleware`, err); - return res.status(500).json({ message: "Internal Server Error" }); + logger.error('Error in auth middleware', err); + res.status(500).json({ message: 'Internal Server Error' }); + return; } }; \ No newline at end of file diff --git a/src/api/routes/admin-auth.routes.ts b/src/api/routes/admin-auth.routes.ts new file mode 100644 index 0000000..9f4fc63 --- /dev/null +++ b/src/api/routes/admin-auth.routes.ts @@ -0,0 +1,29 @@ +/** + * Admin Authentication & Setup Routes + */ + +import { Router } from 'express'; +import { + signupAdmin, + loginAdmin, + logoutAdmin, + getAuthStatus, +} from '../controllers/admin-auth.controller.js'; + +const router = Router(); + +// 1. Signup / Setup - Initial administrator account creation (only allowed if no admin exists) +router.post('/signup', signupAdmin); +router.post('/setup', signupAdmin); + +// 2. Login - Verify administrator credentials and set session cookie +router.post('/login', loginAdmin); + +// 3. Logout - Clear session cookie +router.post('/logout', logoutAdmin); + +// Check if admin credentials are configured (used by onboarding / MCP server) +router.get('/status', getAuthStatus); + + +export default router; diff --git a/src/api/routes/api-key.routes.ts b/src/api/routes/api-key.routes.ts new file mode 100644 index 0000000..2250424 --- /dev/null +++ b/src/api/routes/api-key.routes.ts @@ -0,0 +1,31 @@ +/** + * API Key Management Routes + */ + +import { Router } from 'express'; +import { + listApiKeys, + createApiKey, + getApiKeyUsage, + revokeApiKey, + deleteApiKey, +} from '../controllers/api-key.controller.js'; + +const router = Router(); + +// GET /api/keys - List all keys +router.get('/', listApiKeys); + +// POST /api/keys - Create a new key +router.post('/', createApiKey); + +// GET /api/keys/:id - Get usage and details for a key +router.get('/:id', getApiKeyUsage); + +// POST /api/keys/:id/revoke - Revoke a key +router.post('/:id/revoke', revokeApiKey); + +// DELETE /api/keys/:id - Delete a key +router.delete('/:id', deleteApiKey); + +export default router; diff --git a/src/api/routes/channel-routing.routes.ts b/src/api/routes/channel-routing.routes.ts new file mode 100644 index 0000000..2592505 --- /dev/null +++ b/src/api/routes/channel-routing.routes.ts @@ -0,0 +1,22 @@ +/** + * Channel Routing Routes + * + * Endpoints for configuring default and cascading fallback provider chains + */ + +import { Router } from 'express'; +import { + listChannelRoutings, + getChannelRouting, + setChannelRouting, + deleteChannelRouting, +} from '../controllers/channel-routing.controller.js'; + +const router = Router(); + +router.get('/', listChannelRoutings); +router.get('/:channel', getChannelRouting); +router.put('/:channel', setChannelRouting); +router.delete('/:channel', deleteChannelRouting); + +export default router; diff --git a/src/api/routes/health.routes.ts b/src/api/routes/health.routes.ts new file mode 100644 index 0000000..bde14d3 --- /dev/null +++ b/src/api/routes/health.routes.ts @@ -0,0 +1,15 @@ +import { Router } from 'express'; +import { getHealth, getLiveness, getReadiness } from '../controllers/health.controller.js'; + +const router = Router(); + +// GET /api/health +router.get('/health', getHealth); + +// GET /api/healthz (liveness) +router.get('/healthz', getLiveness); + +// GET /api/readyz (readiness) +router.get('/readyz', getReadiness); + +export default router; diff --git a/src/api/routes/plugins.routes.ts b/src/api/routes/plugins.routes.ts index e0adaa8..a16d296 100644 --- a/src/api/routes/plugins.routes.ts +++ b/src/api/routes/plugins.routes.ts @@ -1,18 +1,34 @@ /** * Plugins Routes * - * Exposes plugin metadata for dashboard + * Exposes plugin metadata, installation, update, and uninstallation endpoints */ import { Router } from 'express'; -import { getPluginsMetadata } from '../controllers/plugins.controller.js'; +import { + getPluginsMetadata, + listInstalledPlugins, + listPluginCatalog, + installPlugin, + changePluginVersion, + uninstallPlugin, + getNpmAuth, + saveNpmAuth, + deleteNpmAuth, +} from '../controllers/plugins.controller.js'; const router = Router(); -/** - * GET /plugins - * Returns available channels and their provider schemas - */ router.get('/', getPluginsMetadata); +router.get('/installed', listInstalledPlugins); +router.get('/catalog/:category', listPluginCatalog); +router.get('/npm-auth', getNpmAuth); +router.post('/npm-auth', saveNpmAuth); +router.delete('/npm-auth', deleteNpmAuth); +router.post('/install', installPlugin); +router.put('/version', changePluginVersion); +router.delete('/:scope/:package', uninstallPlugin); +router.delete('/:package', uninstallPlugin); +router.delete('/', uninstallPlugin); export default router; diff --git a/src/api/routes/providers.routes.ts b/src/api/routes/providers.routes.ts new file mode 100644 index 0000000..467bb6a --- /dev/null +++ b/src/api/routes/providers.routes.ts @@ -0,0 +1,32 @@ +/** + * Providers Routes + * + * Endpoints for managing provider instances and testing connections + */ + +import { Router } from 'express'; +import { + listProviders, + getProvider, + createProvider, + updateProvider, + deleteProvider, + testProvider, + getProvidersRateLimits, + getProviderRateLimit, + resetProviderRateLimit, +} from '../controllers/providers.controller.js'; + +const router = Router(); + +router.get('/', listProviders); +router.post('/test', testProvider); +router.get('/rate-limits', getProvidersRateLimits); +router.get('/:id/rate-limit', getProviderRateLimit); +router.post('/:id/rate-limit/reset', resetProviderRateLimit); +router.get('/:id', getProvider); +router.post('/', createProvider); +router.put('/:id', updateProvider); +router.delete('/:id', deleteProvider); + +export default router; diff --git a/src/api/routes/settings.routes.ts b/src/api/routes/settings.routes.ts new file mode 100644 index 0000000..4d6a75d --- /dev/null +++ b/src/api/routes/settings.routes.ts @@ -0,0 +1,19 @@ +/** + * Dynamic Operational Settings Routes + */ + +import { Router } from 'express'; +import { getSettings, updateSettings, resetSettings } from '../controllers/settings.controller.js'; + +const router = Router(); + +// GET /api/settings - Retrieve current active settings +router.get('/', getSettings); + +// PUT /api/settings - Update operational settings +router.put('/', updateSettings); + +// POST /api/settings/reset - Reset operational settings to system defaults +router.post('/reset', resetSettings); + +export default router; diff --git a/src/api/routes/webhook.routes.ts b/src/api/routes/webhook.routes.ts new file mode 100644 index 0000000..b8d9765 --- /dev/null +++ b/src/api/routes/webhook.routes.ts @@ -0,0 +1,12 @@ +/** + * Webhook Routes + */ + +import { Router } from 'express'; +import { handleWebhookCallback } from '../controllers/webhook.controller.js'; + +const router = Router(); + +router.post('/', handleWebhookCallback); + +export default router; diff --git a/src/api/server.ts b/src/api/server.ts index 0b4d26d..aa5c076 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -2,22 +2,47 @@ import express from 'express'; import type { Request, Response } from 'express'; import { env } from '@src/config/env.config.js'; import { connectMongoDB } from '@src/config/db.config.js'; +import { connectRedis } from '@src/config/redis.config.js'; import { exit } from 'process'; import notification_router from './routes/notification.routes.js'; import plugins_router from './routes/plugins.routes.js'; +import providers_router from './routes/providers.routes.js'; +import channel_routing_router from './routes/channel-routing.routes.js'; import admin_channels_router from './routes/admin-channels.routes.js'; -import notification_templates_router from '@src/api/routes/notification_templates.routes.js'; +import notification_templates_router from './routes/notification_templates.routes.js'; import notifications_management_router from './routes/notifications-management.routes.js'; import alerts_router from './routes/alerts.routes.js'; import dashboard_router from './routes/dashboard.routes.js'; +import settings_router from './routes/settings.routes.js'; +import admin_auth_router from './routes/admin-auth.routes.js'; +import webhook_router from './routes/webhook.routes.js'; +import api_keys_router from './routes/api-key.routes.js'; +import health_router from './routes/health.routes.js'; +import { + getApiReadinessChecks, + getApiLivenessChecks, + setAppShuttingDown +} from './controllers/health.controller.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; +import { disconnectRedis } from '@src/config/redis.config.js'; +import { createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeServer } from '@src/types/types.js'; import { auth_middleware } from './middlewares/auth_middleware.js'; +import { ApiKeyUsageAggregator } from './utils/api-key-cache.js'; import http from 'http'; import helmet from 'helmet'; import cors from 'cors'; -import { createTopics } from '@src/config/kafka.config.js'; +import { createTopics, buildKafkaTopicsFromDatabase } from '@src/config/kafka.config.js'; import { apiLogger as logger } from '@src/workers/utils/logger.js'; -import { buildKafkaTopics } from '@src/config/kafka.config.js'; -import { loadProvidersFromEnv } from '@src/plugins/index.js'; +import { + PluginSyncService, + PluginManagerService, + NpmAuthService, + ProviderManagerService, + ChannelRoutingService, + YamlMigrator, + loadProvidersFromDatabase, +} from '@src/plugins/index.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; //Import the admin channel provider files here for them to self-register @@ -25,6 +50,8 @@ import "@src/admin-alerts/channels/discord.channel.js"; import "@src/admin-alerts/channels/telegram.channel.js"; const app = express(); +let isShuttingDown = false; +let probeServer: HealthProbeServer | null = null; //implement rate limiter with REDIS later @@ -32,74 +59,130 @@ app.use(express.json({ limit: '1mb' })); app.use(helmet()); -app.use(cors({ origin: "*" })); //allows all origins +app.use(cors({ + origin: true, + credentials: true, +})); -app.get("/api", (req: Request, res: Response) => { +// check whether the BASE_PATH is a valid one +let BASE_PATH = env.BASE_PATH; +if (!(BASE_PATH.length > 0 && BASE_PATH.startsWith("/"))) { + BASE_PATH = ""; +} +if (BASE_PATH.endsWith("/")) { + BASE_PATH = BASE_PATH.slice(0, BASE_PATH.length - 1); +} + +app.get(`${BASE_PATH}/api`, (req: Request, res: Response) => { res.json({ info: "Notification Service is running" }); return; }); -// Health check endpoint for Docker/Kubernetes -app.get("/api/health", (req: Request, res: Response) => { - res.status(200).json({ - status: "healthy", - timestamp: new Date().toISOString() - }); - return; -}); - -app.use('/api/notification', auth_middleware, notification_router); -app.use('/api/notifications', auth_middleware, notifications_management_router); -app.use('/api/alerts', auth_middleware, alerts_router); -app.use('/api/dashboard', auth_middleware, dashboard_router); -app.use('/api/plugins', auth_middleware, plugins_router); -app.use('/api/admin-channels', auth_middleware, admin_channels_router); -app.use('/api/templates', auth_middleware, notification_templates_router); +// Health check endpoints for Docker/Kubernetes and API consumers +app.use(`${BASE_PATH}/api`, health_router); + +app.use(`${BASE_PATH}/api/notification`, auth_middleware, notification_router); +app.use(`${BASE_PATH}/api/notifications`, auth_middleware, notifications_management_router); +app.use(`${BASE_PATH}/api/alerts`, auth_middleware, alerts_router); +app.use(`${BASE_PATH}/api/dashboard`, auth_middleware, dashboard_router); +app.use(`${BASE_PATH}/api/plugins`, auth_middleware, plugins_router); +app.use(`${BASE_PATH}/api/providers`, auth_middleware, providers_router); +app.use(`${BASE_PATH}/api/channels/routing`, auth_middleware, channel_routing_router); +app.use(`${BASE_PATH}/api/admin-channels`, auth_middleware, admin_channels_router); +app.use(`${BASE_PATH}/api/templates`, auth_middleware, notification_templates_router); +app.use(`${BASE_PATH}/api/settings`, auth_middleware, settings_router); +app.use(`${BASE_PATH}/api/keys`, auth_middleware, api_keys_router); +app.use(`${BASE_PATH}/api/auth`, admin_auth_router); +app.use(`${BASE_PATH}/api/webhook`, webhook_router); const start_server = async () => { try { + // Fail startup immediately if JWT_SECRET is missing or insecure + void env.JWT_SECRET; + const db = await connectMongoDB(); logger.success("Successfully connected to MongoDB"); - // Load plugins from simplens.config.yaml - // Initialize: false is important here because the API service doesn't need to connect to providers (e.g. SMTP), - // it only needs the metadata/schemas to serve the dashboard. - logger.info('Loading plugins from configuration (metadata only)...'); - await loadProvidersFromEnv({ initialize: false }); + // 0. Initialize Dynamic Configuration & seed defaults into MongoDB + await dynamicConfig.initialize(true); - // Create Kafka topics dynamically from config - const topics = buildKafkaTopics(); + // 1. Connect Redis & initialize subscriber for plugins + await connectRedis(); + await PluginSyncService.startSubscriber(); + NpmAuthService.registerSyncHandlers(); + await NpmAuthService.syncNpmrc(); + PluginManagerService.registerSyncHandlers(); + ProviderManagerService.registerSyncHandlers(); + ChannelRoutingService.registerSyncHandlers(); + + // 2. Check if MongoDB has providers; if empty and YAML exists, migrate it + await YamlMigrator.migrateYamlIfNeeded(); + + // 3. Load provider plugins from MongoDB (metadata only for API server) + logger.info('Loading plugins from MongoDB (metadata only)...'); + await loadProvidersFromDatabase({ initialize: false }); + + // 4. Create Kafka topics dynamically from MongoDB channel routing + const topics = await buildKafkaTopicsFromDatabase(); await createTopics(topics); const server = http.createServer(app); - server.listen(env.PORT, () => logger.success(`Notification Service running at http://localhost:${env.PORT}`)); + server.keepAliveTimeout = 65000; + server.headersTimeout = 66000; + server.listen(env.PORT, () => logger.success(`Notification Service running at http://localhost:${env.PORT}${BASE_PATH}`)); + + // 5. Start health probe server for Kubernetes + probeServer = createHealthProbeServer({ + serviceName: 'api', + readinessChecks: getApiReadinessChecks(), + livenessChecks: getApiLivenessChecks() + }); + await probeServer.start(); + const gracefulShutdown = async (err?: Error, reason?: string) => { - logger.error('Shutting down server', { reason: reason ?? '', error: err?.message ?? '' }); + if (isShuttingDown) return; + isShuttingDown = true; + setAppShuttingDown(true); + logger.info(`Shutting down server (reason: ${reason ?? 'signal'})`); try { - server?.close(() => { - logger.info('HTTP server closed'); - }); + if (probeServer) { + await probeServer.stop(); + } + if (server) { + await new Promise((resolve) => { + server.close(() => { + logger.info('HTTP server closed'); + resolve(); + }); + }); + } + await ApiKeyUsageAggregator.flush(); + await PluginSyncService.stopSubscriber(); + await disconnectRedis(); await db.disconnect(); } catch (e) { logger.error('Error during graceful shutdown', e); } finally { - process.exit(1); + process.exit(err ? 1 : 0); } }; + process.on('SIGTERM', () => void gracefulShutdown(undefined, 'SIGTERM')); + process.on('SIGINT', () => void gracefulShutdown(undefined, 'SIGINT')); + process.on('uncaughtException', (err) => { logger.error('Uncaught exception:', err); void AdminAlertService.sendAlert('service_health', `🔴 UNCAUGHT EXCEPTION IN API SERVER\n` + - `Error: ${err.message}\n` + - `Stack: ${err.stack?.split('\n').slice(0, 3).join('\n')}\n` + + `Error: ${err instanceof Error ? err.message : String(err)}\n` + + `Stack: ${err?.stack?.split('\n').slice(0, 3).join('\n') || "undefined"}\n` + `Action: Review error handling. Check for async operations without try-catch.`, { severity: 'critical' }); - gracefulShutdown(err, 'uncaughtException'); + void gracefulShutdown(err, 'uncaughtException'); }); process.on('unhandledRejection', (reason) => { @@ -111,7 +194,7 @@ const start_server = async () => { `Action: Add .catch() handlers to promises. Review async/await error handling.`, { severity: 'critical' }); - gracefulShutdown(undefined, 'unhandledRejection'); + void gracefulShutdown(undefined, 'unhandledRejection'); }); server.on('error', (err) => { @@ -122,7 +205,7 @@ const start_server = async () => { `Port: ${env.PORT}\n` + `Action: Check if port is in use. Review server logs for stack trace.`, { severity: 'critical' }); - gracefulShutdown(err, 'serverError'); + void gracefulShutdown(err, 'serverError'); }); } catch (err) { diff --git a/src/api/utils/api-key-cache.ts b/src/api/utils/api-key-cache.ts new file mode 100644 index 0000000..e24433c --- /dev/null +++ b/src/api/utils/api-key-cache.ts @@ -0,0 +1,262 @@ +/** + * In-Memory API Key Cache & Debounced Usage Aggregator + * + * Provides high-performance in-memory caching of validated API keys and + * debounced batch aggregation of usage metrics to prevent MongoDB document lock + * contention and connection pool exhaustion under heavy concurrency. + */ + +import api_key_model from '@src/database/models/api-key.models.js'; +import type { ApiKeyDoc, ApiKeyCacheEntry, PendingApiKeyUsage } from '@src/types/types.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +const DEFAULT_CACHE_TTL_MS = 60 * 1000; // 60 seconds +const DEFAULT_FLUSH_INTERVAL_MS = 1000; // 1 second +const DEFAULT_MAX_BATCH_THRESHOLD = 200; // flush early if pending requests exceed this threshold + +export class ApiKeyCache { + private static cacheByHash: Map = new Map(); + private static keyIdToHash: Map = new Map(); + private static ttlMs: number = DEFAULT_CACHE_TTL_MS; + + /** + * Set cache TTL in milliseconds (useful for testing) + */ + public static setTtlMs(ttl: number): void { + this.ttlMs = ttl; + } + + /** + * Retrieve active API key document from memory cache if not expired. + */ + public static get(tokenHash: string): ApiKeyDoc | null { + const entry = this.cacheByHash.get(tokenHash); + if (!entry) return null; + + if (Date.now() - entry.cachedAt > this.ttlMs) { + this.cacheByHash.delete(tokenHash); + this.keyIdToHash.delete(entry.keyDoc.key_id); + return null; + } + + return entry.keyDoc; + } + + /** + * Cache validated API key document in memory. + */ + public static set(tokenHash: string, keyDoc: ApiKeyDoc): void { + this.cacheByHash.set(tokenHash, { + keyDoc, + cachedAt: Date.now(), + }); + this.keyIdToHash.set(keyDoc.key_id, tokenHash); + } + + /** + * Invalidate cached API key by key_id or tokenHash. + */ + public static invalidate(keyIdOrHash: string): void { + if (this.cacheByHash.has(keyIdOrHash)) { + const entry = this.cacheByHash.get(keyIdOrHash); + if (entry) { + this.keyIdToHash.delete(entry.keyDoc.key_id); + } + this.cacheByHash.delete(keyIdOrHash); + return; + } + + const hash = this.keyIdToHash.get(keyIdOrHash); + if (hash) { + this.cacheByHash.delete(hash); + this.keyIdToHash.delete(keyIdOrHash); + } + } + + /** + * Clear all cached keys. + */ + public static clear(): void { + this.cacheByHash.clear(); + this.keyIdToHash.clear(); + } +} + +export class ApiKeyUsageAggregator { + private static pendingUsage: Map = new Map(); + private static flushTimer: NodeJS.Timeout | null = null; + private static isFlushing = false; + + private static getOrCreate(keyId: string): PendingApiKeyUsage { + let entry = this.pendingUsage.get(keyId); + if (!entry) { + entry = { + total_requests: 0, + total_notifications: 0, + by_channel: {}, + last_used_at: new Date(), + }; + this.pendingUsage.set(keyId, entry); + } + return entry; + } + + private static isTestEnvironment(): boolean { + return process.env.NODE_ENV === 'test' || Boolean(process.env.VITEST) || Boolean(process.env.VITEST_WORKER_ID); + } + + private static scheduleFlush(): void { + if (this.isTestEnvironment()) { + void this.flush(); + return; + } + + if (!this.flushTimer) { + this.flushTimer = setTimeout(() => { + this.flushTimer = null; + void this.flush(); + }, DEFAULT_FLUSH_INTERVAL_MS); + if (this.flushTimer.unref) { + this.flushTimer.unref(); + } + } + } + + /** + * Record an incoming API request against an API key in memory. + */ + public static recordRequest(keyId: string): void { + if (!keyId) return; + + if (this.isTestEnvironment()) { + void api_key_model.updateOne( + { key_id: keyId }, + { + $set: { 'usage.last_used_at': new Date() }, + $inc: { 'usage.total_requests': 1 }, + } + ).catch(() => {}); + return; + } + + const entry = this.getOrCreate(keyId); + entry.total_requests += 1; + entry.last_used_at = new Date(); + + if (entry.total_requests >= DEFAULT_MAX_BATCH_THRESHOLD) { + void this.flush(); + } else { + this.scheduleFlush(); + } + } + + /** + * Record processed notifications against an API key in memory. + */ + public static recordNotifications( + keyId: string, + channelCounts: Record, + createdCount: number + ): void { + if (!keyId || createdCount <= 0) return; + + if (this.isTestEnvironment()) { + const incObj: Record = { + 'usage.total_notifications': createdCount, + }; + for (const [channel, count] of Object.entries(channelCounts)) { + incObj[`usage.by_channel.${channel}`] = count; + } + + void api_key_model.updateOne( + { key_id: keyId }, + { + $set: { 'usage.last_used_at': new Date() }, + $inc: incObj, + } + ).catch((err: unknown) => { + logger.warn(`Failed to update usage for API key ${keyId}`, { + error: err instanceof Error ? err.message : String(err), + }); + }); + return; + } + + const entry = this.getOrCreate(keyId); + entry.total_notifications += createdCount; + entry.last_used_at = new Date(); + + for (const [channel, count] of Object.entries(channelCounts)) { + entry.by_channel[channel] = (entry.by_channel[channel] || 0) + count; + } + + if (entry.total_notifications >= DEFAULT_MAX_BATCH_THRESHOLD) { + void this.flush(); + } else { + this.scheduleFlush(); + } + } + + /** + * Flush all buffered usage statistics atomically to MongoDB. + */ + public static async flush(): Promise { + if (this.isFlushing || this.pendingUsage.size === 0) { + return; + } + + if (this.flushTimer) { + clearTimeout(this.flushTimer); + this.flushTimer = null; + } + + this.isFlushing = true; + + try { + while (this.pendingUsage.size > 0) { + const snapshot = new Map(this.pendingUsage); + this.pendingUsage.clear(); + + const updates: Promise[] = []; + + for (const [keyId, usage] of snapshot.entries()) { + const incObj: Record = {}; + + if (usage.total_requests > 0) { + incObj['usage.total_requests'] = usage.total_requests; + } + if (usage.total_notifications > 0) { + incObj['usage.total_notifications'] = usage.total_notifications; + } + for (const [ch, count] of Object.entries(usage.by_channel)) { + if (count > 0) { + incObj[`usage.by_channel.${ch}`] = count; + } + } + + if (Object.keys(incObj).length > 0) { + updates.push( + api_key_model.updateOne( + { key_id: keyId }, + { + $set: { 'usage.last_used_at': usage.last_used_at }, + $inc: incObj, + } + ).catch((err: unknown) => { + logger.warn(`Failed to flush buffered usage for API key ${keyId}`, { + error: err instanceof Error ? err.message : String(err), + }); + }) + ); + } + } + + await Promise.all(updates); + } + } catch (err) { + logger.error('Error during API key usage flush', err); + } finally { + this.isFlushing = false; + } + } +} diff --git a/src/api/utils/jwt.utils.ts b/src/api/utils/jwt.utils.ts new file mode 100644 index 0000000..99398fb --- /dev/null +++ b/src/api/utils/jwt.utils.ts @@ -0,0 +1,56 @@ +/** + * Admin JWT Utilities + * + * Handles generation and verification of standard RFC 7519 JSON Web Tokens (HS256) + * for administrator authentication between Dashboard and SimpleNS Core API. + */ + +import jwt from 'jsonwebtoken'; +import { env } from '@src/config/env.config.js'; +import type { AdminJwtPayload } from '@src/types/types.js'; +import { apiLogger as logger } from '@src/workers/utils/logger.js'; + +const DEFAULT_EXPIRY = '7d'; + +/** + * Generate a signed JWT for the authenticated admin account. + */ +export function generateAdminJwt(admin: { id: string; username: string }): string { + const payload: AdminJwtPayload = { + sub: admin.id, + username: admin.username, + role: 'admin', + }; + + return jwt.sign(payload, env.JWT_SECRET, { + expiresIn: DEFAULT_EXPIRY, + algorithm: 'HS256', + }); +} + +/** + * Verify and decode an Admin JWT. + * Returns decoded payload if valid, null otherwise. + */ +export function verifyAdminJwt(token: string): AdminJwtPayload | null { + try { + const decoded = jwt.verify(token, env.JWT_SECRET, { + algorithms: ['HS256'], + }) as AdminJwtPayload; + + if (decoded && decoded.role === 'admin' && decoded.sub) { + return decoded; + } + + return null; + } catch (err) { + if (err instanceof jwt.TokenExpiredError) { + logger.warn('Admin JWT expired', { expiredAt: err.expiredAt }); + } else if (err instanceof jwt.JsonWebTokenError) { + logger.warn('Invalid Admin JWT signature or format', { message: err.message }); + } else { + logger.error('Unexpected error verifying Admin JWT', err); + } + return null; + } +} diff --git a/src/api/utils/utils.ts b/src/api/utils/utils.ts index 6c38d2a..a24025c 100644 --- a/src/api/utils/utils.ts +++ b/src/api/utils/utils.ts @@ -13,15 +13,17 @@ import { getTopicForChannel, CORE_TOPICS, delayed_notification_topic, + SimpleNSProvider, + ApiKeyDoc, } from "@src/types/types.js"; import { NOTIFICATION_STATUS, OUTBOX_STATUS } from "@src/types/types.js"; import mongoose from "mongoose"; import notification_model from "@src/database/models/notification.models.js"; import outbox_model from "@src/database/models/outbox.models.js"; +import { ApiKeyUsageAggregator } from "./api-key-cache.js"; import { apiLogger as logger } from "@src/workers/utils/logger.js"; import { PluginRegistry } from "@src/plugins/index.js"; import notification_template_model from "@src/database/models/notification-template.models.js"; -import { SimpleNSProvider } from "@src/plugins/interfaces/provider.types.js"; import { ZodError } from "zod"; const preloadTemplatesByIds = async ( @@ -60,6 +62,7 @@ const preloadTemplatesByIds = async ( */ export const convert_notification_request_to_notification_schema = async ( data: notification_request, + apiKeyId?: string, ): Promise => { const notifications: notification[] = []; @@ -128,6 +131,7 @@ export const convert_notification_request_to_notification_schema = async ( status: NOTIFICATION_STATUS.pending, scheduled_at: data.scheduled_at, retry_count: 0, + api_key_id: apiKeyId, created_at: new Date(), }; @@ -142,6 +146,7 @@ export const convert_notification_request_to_notification_schema = async ( */ export const convert_batch_notification_schema_to_notification_schema = async ( data: batch_notification_request, + apiKeyId?: string, ): Promise => { const notifications: notification[] = []; @@ -213,6 +218,7 @@ export const convert_batch_notification_schema_to_notification_schema = async ( status: NOTIFICATION_STATUS.pending, scheduled_at: data.scheduled_at, retry_count: 0, + api_key_id: apiKeyId, created_at: new Date(), }; notifications.push(notification_obj); @@ -406,20 +412,69 @@ export const process_notifications = async ( session.startTransaction(); const notification_ids: mongoose.Types.ObjectId[] = []; + const notifications_to_insert: InstanceType[] = []; const outbox_entries: outbox[] = []; const duplicate_keys: { request_id: string; channel: string }[] = []; - for (const notification of notifications) { - // Check for duplicates (include session for transactional read) - const existingNotification = await notification_model - .findOne({ - request_id: notification.request_id, - channel: notification.channel, - status: { $ne: NOTIFICATION_STATUS.failed }, // Allow retrying failed notifications + const existingDuplicateSet = new Set(); + + if (notifications.length > 1) { + const pairs = notifications.map((n) => ({ + request_id: n.request_id, + channel: n.channel, + })); + const existingDocs = await notification_model + .find({ + $or: pairs, + status: { + $in: [ + NOTIFICATION_STATUS.pending, + NOTIFICATION_STATUS.processing, + NOTIFICATION_STATUS.delivered, + ], + }, }) - .session(session); + .select('request_id channel') + .session(session) + .lean(); + + for (const doc of existingDocs) { + existingDuplicateSet.add(`${doc.request_id}:${doc.channel}`); + } + } - if (existingNotification) { + for (const notification of notifications) { + let isDuplicate = false; + + if (notifications.length > 1) { + isDuplicate = existingDuplicateSet.has( + `${notification.request_id}:${notification.channel}`, + ); + } else { + const query = notification_model + .findOne({ + request_id: notification.request_id, + channel: notification.channel, + status: { + $in: [ + NOTIFICATION_STATUS.pending, + NOTIFICATION_STATUS.processing, + NOTIFICATION_STATUS.delivered, + ], + }, + }) + .session(session); + + const existingNotification = + typeof (query as unknown as { select?: (fields: string) => { lean: () => Promise } }).select === 'function' + ? await (query as unknown as { select: (fields: string) => { lean: () => Promise } }) + .select('_id') + .lean() + : await query; + isDuplicate = Boolean(existingNotification); + } + + if (isDuplicate) { duplicate_keys.push({ request_id: notification.request_id as string, channel: notification.channel, @@ -427,12 +482,11 @@ export const process_notifications = async ( continue; } - // Create notification document within transaction + // Create notification document within transaction (generates _id) const notification_doc = new notification_model(notification); - await notification_doc.save({ session }); - const notification_id = notification_doc._id as mongoose.Types.ObjectId; notification_ids.push(notification_id); + notifications_to_insert.push(notification_doc); // Create outbox entry const outbox_entry = convert_notification_schema_to_outbox_schema( @@ -456,9 +510,12 @@ export const process_notifications = async ( ); } - // Insert outbox entries in bulk within transaction - if (outbox_entries.length > 0) { - await outbox_model.insertMany(outbox_entries, { session }); + // Insert notifications and outbox entries concurrently within transaction + if (notifications_to_insert.length > 0) { + await Promise.all([ + notification_model.insertMany(notifications_to_insert, { session, ordered: false }), + outbox_model.insertMany(outbox_entries, { session, ordered: false }), + ]); } // Commit transaction - both succeed or both fail @@ -537,3 +594,23 @@ export const validateContentSchema = ( ); } }; + +/** + * Update notification usage counters for an API key asynchronously. + */ +export const updateApiKeyNotificationUsage = ( + apiKey: ApiKeyDoc | undefined, + notifications: notification[], + createdCount: number, +): void => { + if (!apiKey || createdCount <= 0) { + return; + } + + const channelCounts: Record = {}; + for (const n of notifications) { + channelCounts[n.channel] = (channelCounts[n.channel] || 0) + 1; + } + + ApiKeyUsageAggregator.recordNotifications(apiKey.key_id, channelCounts, createdCount); +}; diff --git a/src/config/db.config.ts b/src/config/db.config.ts index 4d7856a..0a15103 100644 --- a/src/config/db.config.ts +++ b/src/config/db.config.ts @@ -5,7 +5,12 @@ import { AdminAlertService } from "@src/admin-alerts/admin-alert.service.js"; export const connectMongoDB = async ()=>{ try{ - const db = await mongoose.connect(env.MONGO_URI); + const db = await mongoose.connect(env.MONGO_URI, { + maxPoolSize: 300, + minPoolSize: 20, + maxConnecting: 20, + serverSelectionTimeoutMS: 10000, + }); await db.connection.syncIndexes(); logger.info("MongoDB indexes synced"); diff --git a/src/config/dynamic-config.service.ts b/src/config/dynamic-config.service.ts new file mode 100644 index 0000000..8205874 --- /dev/null +++ b/src/config/dynamic-config.service.ts @@ -0,0 +1,265 @@ +/** + * Dynamic Operational Configuration Service + * + * Manages runtime system settings stored in MongoDB and live hot-reloaded + * across all running containers via Redis Pub/Sub without process restarts. + */ + +import { EventEmitter } from 'events'; +import system_config_model from '@src/database/models/system-config.models.js'; +import { getRedisClient } from './redis.config.js'; +import { configSyncLogger as logger, setLogLevel } from '@src/workers/utils/logger.js'; +import { operationalSettingsSchema } from '@src/types/schemas.js'; +import type { + OperationalSettings, + PartialOperationalSettings, + SystemConfigSyncAction, + SystemConfigSyncMessage, +} from '@src/types/types.js'; +import type { Redis as RedisType } from 'ioredis'; + +export const CONFIG_SYNC_CHANNEL = 'simplens:system:config:sync'; + +export const SYSTEM_DEFAULT_OPERATIONAL_SETTINGS: OperationalSettings = { + api: { + max_batch_req_limit: 1000, + }, + worker: { + outbox_poll_interval_ms: 5000, + outbox_cleanup_interval_ms: 60000, + outbox_batch_size: 100, + outbox_retention_ms: 300000, + outbox_claim_timeout_ms: 30000, + }, + retry: { + max_retry_count: 5, + idempotency_ttl_seconds: 86400, + processing_ttl_seconds: 120, + rate_limit_retry_delay_ms: 5000, + }, + delayed: { + delayed_poll_interval_ms: 1000, + delayed_batch_size: 10, + max_poller_retries: 3, + }, + recovery: { + recovery_poll_interval_ms: 60000, + processing_stuck_threshold_ms: 300000, + pending_stuck_threshold_ms: 300000, + recovery_batch_size: 50, + recovery_claim_timeout_ms: 60000, + cleanup_resolved_alerts_retention_ms: 86400000, + cleanup_processed_status_outbox_retention_ms: 86400000, + }, + logging: { + log_level: 'info', + log_to_file: true, + }, +}; + +export class DynamicConfigService extends EventEmitter { + private static instance: DynamicConfigService; + private settings: OperationalSettings = { ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS }; + private isInitialized = false; + private subClient: RedisType | null = null; + private instanceId: string; + + private constructor() { + super(); + this.instanceId = process.env.HOSTNAME || process.env.WORKER_ID || `instance-${Math.random().toString(36).slice(2, 9)}`; + } + + public static getInstance(): DynamicConfigService { + if (!DynamicConfigService.instance) { + DynamicConfigService.instance = new DynamicConfigService(); + } + return DynamicConfigService.instance; + } + + /** + * Initialize DynamicConfigService + * @param isApiServer - If true, seeds MongoDB with default values if not present, and broadcasts. + */ + public async initialize(isApiServer: boolean = false): Promise { + if (this.isInitialized) return; + + // 1. Subscribe to Redis sync updates + await this.subscribeToUpdates(); + + // 2. Fetch from MongoDB or seed defaults + try { + const doc = await system_config_model.findOne({ key: 'operational_settings' }); + + if (doc?.value) { + const parsed = operationalSettingsSchema.safeParse(doc.value); + if (parsed.success) { + this.settings = parsed.data; + logger.info('Loaded operational settings from MongoDB'); + } else { + logger.warn('Failed to parse operational settings from MongoDB, using defaults', { error: parsed.error }); + this.settings = { ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS }; + } + } else if (isApiServer) { + this.settings = { ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS }; + await system_config_model.create({ + key: 'operational_settings', + value: this.settings, + }); + logger.success('Seeded MongoDB with default operational settings'); + + await this.publishSync('SETTINGS_SEEDED'); + } else { + this.settings = { ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS }; + logger.info('No operational settings found in MongoDB yet; using default in-memory settings'); + } + } catch (err) { + logger.error('Error loading operational settings from MongoDB, using defaults', err); + this.settings = { ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS }; + } + + // Apply log level to all instantiated loggers + setLogLevel(this.settings.logging.log_level); + + this.isInitialized = true; + } + + public get(group: K): OperationalSettings[K] { + return this.settings[group]; + } + + public getAll(): OperationalSettings { + return { ...this.settings }; + } + + public async updateSettings(newSettings: PartialOperationalSettings): Promise { + const merged = { + api: { ...this.settings.api, ...(newSettings.api || {}) }, + worker: { ...this.settings.worker, ...(newSettings.worker || {}) }, + retry: { ...this.settings.retry, ...(newSettings.retry || {}) }, + delayed: { ...this.settings.delayed, ...(newSettings.delayed || {}) }, + recovery: { ...this.settings.recovery, ...(newSettings.recovery || {}) }, + logging: { ...this.settings.logging, ...(newSettings.logging || {}) }, + }; + + const validated = operationalSettingsSchema.parse(merged); + + await system_config_model.findOneAndUpdate( + { key: 'operational_settings' }, + { value: validated }, + { upsert: true, returnDocument: 'after' } + ); + + this.settings = validated; + setLogLevel(this.settings.logging.log_level); + + this.emit('change', this.settings); + + // Broadcast sync event to all instances via Redis Pub/Sub + await this.publishSync('SETTINGS_UPDATED', newSettings); + + return this.settings; + } + + public async resetSettings(): Promise { + const validated = operationalSettingsSchema.parse(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS); + + await system_config_model.findOneAndUpdate( + { key: 'operational_settings' }, + { value: validated }, + { upsert: true, returnDocument: 'after' } + ); + + this.settings = validated; + setLogLevel(this.settings.logging.log_level); + + this.emit('change', this.settings); + + await this.publishSync('SETTINGS_RESET'); + + return this.settings; + } + + public async loadFromDatabase(): Promise { + try { + const doc = await system_config_model.findOne({ key: 'operational_settings' }); + if (doc?.value) { + const parsed = operationalSettingsSchema.safeParse(doc.value); + if (parsed.success) { + this.settings = parsed.data; + setLogLevel(this.settings.logging.log_level); + this.emit('change', this.settings); + } + } + } catch (err) { + logger.error('Failed to reload operational settings from MongoDB', err); + } + } + + private async publishSync(action: SystemConfigSyncAction, payload?: PartialOperationalSettings): Promise { + try { + const redis = getRedisClient(); + const message: SystemConfigSyncMessage = { + action, + source_instance_id: this.instanceId, + timestamp: new Date().toISOString(), + payload, + }; + await redis.publish(CONFIG_SYNC_CHANNEL, JSON.stringify(message)); + logger.info(`Published config sync event [${action}] to ${CONFIG_SYNC_CHANNEL}`); + } catch (err) { + logger.error(`Failed to broadcast settings sync event [${action}] to Redis`, err); + } + } + + private async subscribeToUpdates(): Promise { + try { + const baseClient = getRedisClient(); + this.subClient = baseClient.duplicate(); + + // If client status is not ready/connecting, connect + if (this.subClient.status !== 'ready' && this.subClient.status !== 'connecting') { + await this.subClient.connect().catch(() => {}); + } + + await this.subClient.subscribe(CONFIG_SYNC_CHANNEL); + + this.subClient.on('message', async (channel, messageStr) => { + if (channel !== CONFIG_SYNC_CHANNEL) return; + + try { + const message: SystemConfigSyncMessage = JSON.parse(messageStr); + + // Skip self-originated updates + if (message.source_instance_id === this.instanceId) { + return; + } + + logger.info(`Received config sync event [${message.action}] from ${message.source_instance_id}`); + await this.loadFromDatabase(); + } catch (err) { + logger.error('Failed to process settings sync message from Redis', err); + } + }); + } catch (err) { + logger.warn('Failed to subscribe to Redis config sync channel, running with in-memory settings', { + error: err instanceof Error ? err.message : String(err) + }); + } + } + + public async close(): Promise { + if (this.subClient) { + try { + await this.subClient.unsubscribe(CONFIG_SYNC_CHANNEL); + await this.subClient.quit(); + } catch { + // Ignore disconnect errors during teardown + } + this.subClient = null; + } + this.isInitialized = false; + this.removeAllListeners(); + } +} + +export const dynamicConfig = DynamicConfigService.getInstance(); diff --git a/src/config/env.config.ts b/src/config/env.config.ts index 9aa5481..d2651de 100644 --- a/src/config/env.config.ts +++ b/src/config/env.config.ts @@ -1,12 +1,47 @@ import dotenv from 'dotenv'; import { randomUUID } from 'crypto'; +import { dynamicConfig } from './dynamic-config.service.js'; dotenv.config(); // Unique worker/processor ID for this instance (used for distributed locking) const WORKER_ID = process.env.WORKER_ID || `worker-${randomUUID().slice(0, 8)}`; +export const INSECURE_JWT_SECRET_FALLBACK = "simplens-default-jwt-secret-key-change-in-production"; + +/** + * Validates that JWT_SECRET is explicitly configured and cryptographically strong. + * Throws a fatal error if missing, set to insecure default, or too short. + */ +export function validateJwtSecret(customSecret?: string): string { + const secret = (customSecret !== undefined ? customSecret : process.env.JWT_SECRET)?.trim(); + + if (!secret) { + throw new Error( + "FATAL: JWT_SECRET environment variable is missing. SimpleNS requires an explicitly configured JWT secret to secure admin tokens." + ); + } + + if (secret === INSECURE_JWT_SECRET_FALLBACK) { + throw new Error( + "FATAL: JWT_SECRET is set to the default insecure placeholder. You must configure a unique, secure secret." + ); + } + + if (secret.length < 32) { + throw new Error( + `FATAL: JWT_SECRET is too short (${secret.length} characters). It must be at least 32 characters long for cryptographic security.` + ); + } + + return secret; +} + export const env = { + // ======================================================================== + // 1. BOOTSTRAP ENVIRONMENT VARIABLES (from .env / process.env) + // ======================================================================== + // Node Environment NODE_ENV: process.env.NODE_ENV || 'development', @@ -18,49 +53,107 @@ export const env = { // API Server PORT: parseInt(process.env.PORT || "3000"), - NS_API_KEY: (process.env.NS_API_KEY || ""), - MAX_BATCH_REQ_LIMIT: parseInt(process.env.MAX_BATCH_REQ_LIMIT || "1000"), + get JWT_SECRET(): string { + return validateJwtSecret(); + }, DELAYED_PARTITION: parseInt(process.env.DELAYED_PARTITION || "1"), NOTIFICATION_STATUS_PARTITION: parseInt(process.env.NOTIFICATION_STATUS_PARTITION || "1"), // Kafka BROKERS: (process.env.BROKERS?.trim().split(',') || ["localhost:9092"]), - // Background Worker + // Distributed Worker Identity WORKER_ID, - OUTBOX_POLL_INTERVAL_MS: parseInt(process.env.OUTBOX_POLL_INTERVAL_MS || "5000"), - OUTBOX_CLEANUP_INTERVAL_MS: parseInt(process.env.OUTBOX_CLEANUP_INTERVAL_MS || "60000"), - OUTBOX_BATCH_SIZE: parseInt(process.env.OUTBOX_BATCH_SIZE || "100"), - OUTBOX_RETENTION_MS: parseInt(process.env.OUTBOX_RETENTION_MS || "300000"), - OUTBOX_CLAIM_TIMEOUT_MS: parseInt(process.env.OUTBOX_CLAIM_TIMEOUT_MS || "30000"), + + // Logging - Grafana Loki + LOKI_URL: process.env.LOKI_URL || "", + + // Admin Alerts (optional - auto-generated if not provided) + ADMIN_ALERT_ENCRYPTION_KEY: process.env.ADMIN_ALERT_ENCRYPTION_KEY || "", + + // Application Base Path + BASE_PATH: process.env.BASE_PATH || "", + + // ======================================================================== + // 2. DYNAMIC OPERATIONAL VARIABLES (Delegated to DynamicConfigService) + // These reflect live in-memory MongoDB configuration with zero-restart updates. + // ======================================================================== + + // API + get MAX_BATCH_REQ_LIMIT(): number { + return dynamicConfig?.get?.('api')?.max_batch_req_limit ?? 1000; + }, + + // Background Worker + get OUTBOX_POLL_INTERVAL_MS(): number { + return dynamicConfig?.get?.('worker')?.outbox_poll_interval_ms ?? 5000; + }, + get OUTBOX_CLEANUP_INTERVAL_MS(): number { + return dynamicConfig?.get?.('worker')?.outbox_cleanup_interval_ms ?? 60000; + }, + get OUTBOX_BATCH_SIZE(): number { + return dynamicConfig?.get?.('worker')?.outbox_batch_size ?? 100; + }, + get OUTBOX_RETENTION_MS(): number { + return dynamicConfig?.get?.('worker')?.outbox_retention_ms ?? 300000; + }, + get OUTBOX_CLAIM_TIMEOUT_MS(): number { + return dynamicConfig?.get?.('worker')?.outbox_claim_timeout_ms ?? 30000; + }, // Idempotency & Retry - IDEMPOTENCY_TTL_SECONDS: parseInt(process.env.IDEMPOTENCY_TTL_SECONDS || "86400"), - MAX_RETRY_COUNT: parseInt(process.env.MAX_RETRY_COUNT || "5"), - PROCESSING_TTL_SECONDS: parseInt(process.env.PROCESSING_TTL_SECONDS || "120"), + get IDEMPOTENCY_TTL_SECONDS(): number { + return dynamicConfig?.get?.('retry')?.idempotency_ttl_seconds ?? 86400; + }, + get MAX_RETRY_COUNT(): number { + return dynamicConfig?.get?.('retry')?.max_retry_count ?? 5; + }, + get PROCESSING_TTL_SECONDS(): number { + return dynamicConfig?.get?.('retry')?.processing_ttl_seconds ?? 120; + }, + get RATE_LIMIT_RETRY_DELAY_MS(): number { + return dynamicConfig?.get?.('retry')?.rate_limit_retry_delay_ms ?? 5000; + }, // Delayed Worker - MAX_POLLER_RETRIES: parseInt(process.env.MAX_POLLER_RETRIES || "3"), - DELAYED_POLL_INTERVAL_MS: parseInt(process.env.DELAYED_POLL_INTERVAL_MS || "1000"), - DELAYED_BATCH_SIZE: parseInt(process.env.DELAYED_BATCH_SIZE || "10"), + get MAX_POLLER_RETRIES(): number { + return dynamicConfig?.get?.('delayed')?.max_poller_retries ?? 3; + }, + get DELAYED_POLL_INTERVAL_MS(): number { + return dynamicConfig?.get?.('delayed')?.delayed_poll_interval_ms ?? 1000; + }, + get DELAYED_BATCH_SIZE(): number { + return dynamicConfig?.get?.('delayed')?.delayed_batch_size ?? 10; + }, // Recovery Service - RECOVERY_POLL_INTERVAL_MS: parseInt(process.env.RECOVERY_POLL_INTERVAL_MS || "60000"), - PROCESSING_STUCK_THRESHOLD_MS: parseInt(process.env.PROCESSING_STUCK_THRESHOLD_MS || "300000"), - PENDING_STUCK_THRESHOLD_MS: parseInt(process.env.PENDING_STUCK_THRESHOLD_MS || "300000"), - RECOVERY_BATCH_SIZE: parseInt(process.env.RECOVERY_BATCH_SIZE || "50"), + get RECOVERY_POLL_INTERVAL_MS(): number { + return dynamicConfig?.get?.('recovery')?.recovery_poll_interval_ms ?? 60000; + }, + get PROCESSING_STUCK_THRESHOLD_MS(): number { + return dynamicConfig?.get?.('recovery')?.processing_stuck_threshold_ms ?? 300000; + }, + get PENDING_STUCK_THRESHOLD_MS(): number { + return dynamicConfig?.get?.('recovery')?.pending_stuck_threshold_ms ?? 300000; + }, + get RECOVERY_BATCH_SIZE(): number { + return dynamicConfig?.get?.('recovery')?.recovery_batch_size ?? 50; + }, + get RECOVERY_CLAIM_TIMEOUT_MS(): number { + return dynamicConfig?.get?.('recovery')?.recovery_claim_timeout_ms ?? 60000; + }, + get CLEANUP_RESOLVED_ALERTS_RETENTION_MS(): number { + return dynamicConfig?.get?.('recovery')?.cleanup_resolved_alerts_retention_ms ?? 86400000; + }, + get CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS(): number { + return dynamicConfig?.get?.('recovery')?.cleanup_processed_status_outbox_retention_ms ?? 86400000; + }, - // Cleanup Configuration (run as part of recovery cron) - CLEANUP_RESOLVED_ALERTS_RETENTION_MS: parseInt(process.env.CLEANUP_RESOLVED_ALERTS_RETENTION_MS || "86400000"), // 24 hours - CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS: parseInt(process.env.CLEANUP_PROCESSED_STATUS_OUTBOX_RETENTION_MS || "86400000"), // 24 hours - RECOVERY_CLAIM_TIMEOUT_MS: parseInt(process.env.RECOVERY_CLAIM_TIMEOUT_MS || "60000"), // 1 minute - - // Logging - Grafana Loki - LOKI_URL: process.env.LOKI_URL || "", - LOG_LEVEL: process.env.LOG_LEVEL || "info", - LOG_TO_FILE: process.env.LOG_TO_FILE === "true", - - // Admin Alerts (optional - auto-generated if not provided) - // Must be 64-character hex string (32 bytes) if provided - ADMIN_ALERT_ENCRYPTION_KEY: process.env.ADMIN_ALERT_ENCRYPTION_KEY || "", -} \ No newline at end of file + // Logging + get LOG_LEVEL(): string { + return dynamicConfig?.get?.('logging')?.log_level ?? (process.env.LOG_LEVEL || 'info'); + }, + get LOG_TO_FILE(): boolean { + return dynamicConfig?.get?.('logging')?.log_to_file ?? (process.env.LOG_TO_FILE === 'true'); + }, +}; \ No newline at end of file diff --git a/src/config/kafka.config.ts b/src/config/kafka.config.ts index 2fb5c02..c8fd0c5 100644 --- a/src/config/kafka.config.ts +++ b/src/config/kafka.config.ts @@ -52,6 +52,45 @@ export const buildKafkaTopics = (): { topic: string; numPartitions: number; repl return topics; }; +/** + * Build Kafka topics dynamically from MongoDB channel routing records + */ +export const buildKafkaTopicsFromDatabase = async (): Promise => { + const topics: ITopicConfig[] = [ + { + topic: CORE_TOPICS.delayed_notification, + numPartitions: env.DELAYED_PARTITION, + replicationFactor: 1 + }, + { + topic: CORE_TOPICS.notification_status, + numPartitions: env.NOTIFICATION_STATUS_PARTITION, + replicationFactor: 1 + }, + ]; + + try { + const ChannelRouting = (await import('@src/database/models/channel-routing.models.js')).default; + const routings = await ChannelRouting.find().lean(); + if (routings && routings.length > 0) { + for (const r of routings) { + topics.push({ + topic: getTopicForChannel(r.channel), + numPartitions: r.partitions || 6, + replicationFactor: 1, + }); + logger.info(`Configured topic from DB: ${getTopicForChannel(r.channel)} (${r.partitions || 6} partitions)`); + } + return topics; + } + } catch (err) { + logger.warn('Could not query ChannelRouting for Kafka topics, falling back to static config:', { + error: err instanceof Error ? err.message : String(err) + }); + } + + return buildKafkaTopics(); +}; export const createTopics = async (topics: ITopicConfig[]) => { const admin = kafka.admin(); @@ -77,10 +116,19 @@ export const createTopics = async (topics: ITopicConfig[]) => { const currentPartitions = topicMetadata.partitions.length; const desiredPartitions = configuredTopic.numPartitions; - if (currentPartitions !== desiredPartitions) { + if (desiredPartitions > currentPartitions) { + logger.info(`Expanding topic "${topicMetadata.name}" from ${currentPartitions} to ${desiredPartitions} partitions...`); + await admin.createPartitions({ + topicPartitions: [{ + topic: topicMetadata.name, + count: desiredPartitions + }] + }); + logger.success(`Successfully expanded topic "${topicMetadata.name}" to ${desiredPartitions} partitions`); + } else if (desiredPartitions < currentPartitions) { logger.info( - `Topic "${topicMetadata.name}" has ${currentPartitions} partitions but ${desiredPartitions} is configured. ` + - `Partition count cannot be changed after creation. Delete and recreate the topic to apply changes.` + `Topic "${topicMetadata.name}" has ${currentPartitions} partitions (configured: ${desiredPartitions}). ` + + `Kafka partition count cannot be decreased.` ); } } @@ -116,3 +164,77 @@ export const createTopics = async (topics: ITopicConfig[]) => { } }; +/** + * Ensure a channel topic exists in Kafka with at least desiredPartitions + */ +export const ensureChannelTopic = async ( + channel: string, + desiredPartitions: number = 6 +): Promise => { + const topic = getTopicForChannel(channel); + await createTopics([ + { + topic, + numPartitions: desiredPartitions, + replicationFactor: 1, + } + ]); +}; + +/** + * Dynamically expand partition count for a channel topic (or create if missing) + */ +export const expandTopicPartitions = async ( + channel: string, + desiredPartitions: number +): Promise<{ previous: number; current: number }> => { + const admin = kafka.admin(); + const topic = getTopicForChannel(channel); + + try { + await admin.connect(); + const metadata = await admin.fetchTopicMetadata({ topics: [topic] }); + const topicMetadata = metadata.topics.find(t => t.name === topic); + + if (!topicMetadata) { + logger.info(`Topic '${topic}' for channel '${channel}' not found in Kafka, creating with ${desiredPartitions} partitions...`); + await admin.createTopics({ + topics: [{ + topic, + numPartitions: desiredPartitions, + replicationFactor: 1, + }], + validateOnly: false, + timeout: 30000, + }); + logger.success(`Created Kafka topic '${topic}' with ${desiredPartitions} partitions`); + return { previous: 0, current: desiredPartitions }; + } + + const currentPartitions = topicMetadata.partitions.length; + + if (desiredPartitions < currentPartitions) { + throw new Error( + `Kafka partition count cannot be decreased. Current count is ${currentPartitions}, requested ${desiredPartitions}.` + ); + } + + if (desiredPartitions > currentPartitions) { + logger.info(`Expanding Kafka topic '${topic}' from ${currentPartitions} to ${desiredPartitions} partitions...`); + await admin.createPartitions({ + topicPartitions: [{ + topic, + count: desiredPartitions + }] + }); + logger.success(`Expanded Kafka topic '${topic}' to ${desiredPartitions} partitions`); + return { previous: currentPartitions, current: desiredPartitions }; + } + + return { previous: currentPartitions, current: currentPartitions }; + } finally { + await admin.disconnect(); + } +}; + + diff --git a/src/database/models/admin-channel.models.ts b/src/database/models/admin-channel.models.ts index aa66a48..6bc541c 100644 --- a/src/database/models/admin-channel.models.ts +++ b/src/database/models/admin-channel.models.ts @@ -59,9 +59,11 @@ const admin_channel_schema = new mongoose.Schema( } ); -const admin_channel_model = mongoose.model( - 'AdminNotificationChannel', - admin_channel_schema -); +const admin_channel_model = + mongoose.models.AdminNotificationChannel || + mongoose.model( + 'AdminNotificationChannel', + admin_channel_schema + ); export default admin_channel_model; diff --git a/src/database/models/api-key.models.ts b/src/database/models/api-key.models.ts new file mode 100644 index 0000000..615e7ad --- /dev/null +++ b/src/database/models/api-key.models.ts @@ -0,0 +1,80 @@ +/** + * API Key Model + * + * Stores hashed API keys and real-time usage metrics for external service authentication. + */ + +import { type ApiKeyDoc } from '@src/types/types.js'; +import mongoose from 'mongoose'; + +const api_key_schema = new mongoose.Schema( + { + key_id: { + type: String, + required: true, + unique: true, + index: true, + }, + name: { + type: String, + required: true, + }, + type: { + type: String, + enum: ['notification', 'admin'], + default: 'notification', + index: true, + }, + key_prefix: { + type: String, + required: true, + }, + key_hash: { + type: String, + required: true, + unique: true, + index: true, + }, + status: { + type: String, + enum: ['active', 'revoked'], + default: 'active', + index: true, + }, + expires_at: { + type: Date, + default: null, + }, + usage: { + total_requests: { + type: Number, + default: 0, + }, + total_notifications: { + type: Number, + default: 0, + }, + by_channel: { + type: Map, + of: Number, + default: {}, + }, + last_used_at: { + type: Date, + default: null, + }, + }, + }, + { + timestamps: { + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + } +); + +const api_key_model = + mongoose.models?.ApiKey || + mongoose.model('ApiKey', api_key_schema); + +export default api_key_model; diff --git a/src/database/models/channel-routing.models.ts b/src/database/models/channel-routing.models.ts new file mode 100644 index 0000000..1cb5c76 --- /dev/null +++ b/src/database/models/channel-routing.models.ts @@ -0,0 +1,45 @@ +/** + * Channel Routing Model + * Stores default and cascading fallback provider chains, plus Kafka partition targets + */ + +import { type channel_routing_document } from '@src/types/types.js'; +import mongoose from 'mongoose'; + +const channel_routing_schema = new mongoose.Schema( + { + channel: { + type: String, + required: true, + unique: true, + index: true, + }, + default_provider_id: { + type: String, + required: true, + }, + fallback_provider_ids: { + type: [String], + default: [], + }, + partitions: { + type: Number, + default: 6, + min: 1, + }, + }, + { + timestamps: { + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + } +); + +const channel_routing_model = + mongoose.models.ChannelRouting || mongoose.model( + 'ChannelRouting', + channel_routing_schema + ); + +export default channel_routing_model; diff --git a/src/database/models/notification.models.ts b/src/database/models/notification.models.ts index afbf7ce..e03476f 100644 --- a/src/database/models/notification.models.ts +++ b/src/database/models/notification.models.ts @@ -40,6 +40,18 @@ const notification_schema = new mongoose.Schema( type: String, index: true, }, + provider_history: [ + { + provider: { type: String, required: true }, + status: { type: String, enum: ['delivered', 'failed', 'rate_limited'], required: true }, + error_code: { type: String }, + error_message: { type: String }, + retryable: { type: Boolean }, + attempted_at: { type: Date, default: Date.now }, + duration_ms: { type: Number }, + response: { type: mongoose.Schema.Types.Mixed }, + } + ], // Dynamic recipient schema - structure depends on channel recipient: { type: mongoose.Schema.Types.Mixed, @@ -74,6 +86,11 @@ const notification_schema = new mongoose.Schema( type: Number, default: 0, }, + api_key_id: { + type: String, + default: null, + index: true, + }, // Recovery claiming fields for horizontal scalability recovery_claimed_by: { type: String, @@ -110,6 +127,9 @@ notification_schema.index( } ); +// Compound index for fast duplicate checking and status queries +notification_schema.index({ request_id: 1, channel: 1, status: 1 }); + const notification_model = mongoose.model('Notification', notification_schema); export default notification_model; \ No newline at end of file diff --git a/src/database/models/plugin.models.ts b/src/database/models/plugin.models.ts new file mode 100644 index 0000000..57965dd --- /dev/null +++ b/src/database/models/plugin.models.ts @@ -0,0 +1,61 @@ +/** + * Plugin Model + * Stores installed notification plugins with metadata and package version + */ + +import { type plugin_document, type provider_manifest } from '@src/types/types.js'; +import mongoose from 'mongoose'; + +const manifestSubSchema = new mongoose.Schema( + { + name: { type: String, required: true }, + version: { type: String, required: true }, + channel: { type: String, required: true }, + displayName: { type: String, required: true }, + description: { type: String, required: true }, + author: { type: String }, + homepage: { type: String }, + requiredCredentials: [{ type: String, required: true }], + optionalConfig: [{ type: String }], + }, + { _id: false } +); + +const plugin_schema = new mongoose.Schema( + { + name: { + type: String, + required: true, + unique: true, + index: true, + }, + version: { + type: String, + required: true, + }, + status: { + type: String, + enum: ['installed', 'installing', 'failed'], + default: 'installed', + index: true, + }, + error: { + type: String, + }, + manifest: { + type: manifestSubSchema, + required: true, + }, + }, + { + timestamps: { + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + } +); + +const plugin_model = + mongoose.models.Plugin || mongoose.model('Plugin', plugin_schema); + +export default plugin_model; diff --git a/src/database/models/provider.models.ts b/src/database/models/provider.models.ts new file mode 100644 index 0000000..0b7e9d1 --- /dev/null +++ b/src/database/models/provider.models.ts @@ -0,0 +1,69 @@ +/** + * Provider Model + * Stores configured provider instances with encrypted credentials + */ + +import { + type provider_document, + type encrypted_credentials, +} from '@src/types/types.js'; +import mongoose from 'mongoose'; + +// Subdocument schema for key-pair envelope encrypted credentials +const encryptedCredentialsSubSchema = new mongoose.Schema( + { + encrypted_data: { type: String, required: true }, + iv: { type: String, required: true }, + auth_tag: { type: String, required: true }, + encrypted_dek: { type: String, required: true }, + }, + { _id: false } +); + +const provider_schema = new mongoose.Schema( + { + id: { + type: String, + required: true, + unique: true, + index: true, + }, + plugin_name: { + type: String, + required: true, + index: true, + }, + channel: { + type: String, + required: true, + index: true, + }, + enabled: { + type: Boolean, + default: true, + index: true, + }, + credentials: { + type: encryptedCredentialsSubSchema, + required: true, + }, + options: { + type: mongoose.Schema.Types.Mixed, + default: {}, + }, + }, + { + timestamps: { + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + } +); + +// Compound index for priority-based provider routing per channel +provider_schema.index({ channel: 1, 'options.priority': -1 }); + +const provider_model = + mongoose.models.Provider || mongoose.model('Provider', provider_schema); + +export default provider_model; diff --git a/src/database/models/status-outbox.models.ts b/src/database/models/status-outbox.models.ts index 90ce3ac..4dcd8da 100644 --- a/src/database/models/status-outbox.models.ts +++ b/src/database/models/status-outbox.models.ts @@ -21,6 +21,21 @@ const status_outbox_schema = new mongoose.Schema( enum: Object.values(NOTIFICATION_STATUS_SF), required: true, }, + provider: { + type: String, + }, + provider_history: [ + { + provider: { type: String, required: true }, + status: { type: String, enum: ['delivered', 'failed', 'rate_limited'], required: true }, + error_code: { type: String }, + error_message: { type: String }, + retryable: { type: Boolean }, + attempted_at: { type: Date, default: Date.now }, + duration_ms: { type: Number }, + response: { type: mongoose.Schema.Types.Mixed }, + } + ], processed: { type: Boolean, default: false, @@ -46,6 +61,7 @@ const status_outbox_schema = new mongoose.Schema( // Index for efficient polling of unprocessed entries status_outbox_schema.index({ processed: 1, created_at: 1 }); +status_outbox_schema.index({ processed: 1, claimed_by: 1, created_at: 1 }); // Index for claiming stale entries status_outbox_schema.index({ processed: 1, claimed_at: 1 }); diff --git a/src/database/models/system-config.models.ts b/src/database/models/system-config.models.ts index 13afa69..59418a2 100644 --- a/src/database/models/system-config.models.ts +++ b/src/database/models/system-config.models.ts @@ -15,7 +15,7 @@ const system_config_schema = new mongoose.Schema( index: true, }, value: { - type: String, + type: mongoose.Schema.Types.Mixed, required: true, }, }, @@ -27,6 +27,8 @@ const system_config_schema = new mongoose.Schema( } ); -const system_config_model = mongoose.model('SystemConfig', system_config_schema); +const system_config_model = + mongoose.models?.SystemConfig || + mongoose.model('SystemConfig', system_config_schema); export default system_config_model; diff --git a/src/plugins/bootstrap/yaml-migrator.ts b/src/plugins/bootstrap/yaml-migrator.ts new file mode 100644 index 0000000..3dccab5 --- /dev/null +++ b/src/plugins/bootstrap/yaml-migrator.ts @@ -0,0 +1,244 @@ +/** + * YAML Configuration Migrator + * + * Migrates static simplens.config.yaml into MongoDB on first startup if MongoDB is empty. + * Resolves env placeholders in credentials, encrypts them using KeyPairManager, + * ensures npm packages are installed, extracts manifests with package.json versions, + * converts single fallbacks to array chains, and broadcasts SYSTEM_RELOAD. + */ + +import { existsSync, readFileSync } from 'fs'; +import { join } from 'path'; +import { parse as parseYaml } from 'yaml'; +import Plugin from '@src/database/models/plugin.models.js'; +import Provider from '@src/database/models/provider.models.js'; +import ChannelRouting from '@src/database/models/channel-routing.models.js'; +import { encryptCredentials } from '@src/plugins/crypto/keypair-manager.js'; +import { + installNpmPackage, + extractPackageManifest, + PLUGINS_NODE_MODULES, + resolveCredentials, + resolveOptionalConfig, + findConfigFile, +} from '@src/plugins/loader/plugin-fs.js'; +import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; +import type { YamlConfig } from '@src/types/types.js'; + +export class YamlMigrator { + /** + * Check if MongoDB has providers; if empty and config file exists, migrate to MongoDB. + * Returns true if migration was performed, false otherwise. + */ + public static async migrateYamlIfNeeded(): Promise { + const providerCount = await Provider.countDocuments(); + if (providerCount > 0) { + logger.info( + `Found ${providerCount} provider(s) in MongoDB. Dynamic database configuration active, skipping YAML migration.` + ); + return false; + } + + + const configPath = findConfigFile(); + if (!configPath) { + logger.info('No simplens.config.yaml found and MongoDB is empty. Standby mode ready.'); + return false; + } + + logger.info(`MongoDB provider collection is empty. Migrating configuration from ${configPath}...`); + + let rawConfig: YamlConfig; + try { + const fileContent = readFileSync(configPath, 'utf-8'); + if (configPath.endsWith('.json')) { + rawConfig = JSON.parse(fileContent); + } else { + rawConfig = parseYaml(fileContent) as YamlConfig; + } + } catch (err) { + logger.error(`Failed to parse configuration file at ${configPath}:`, err); + return false; + } + + if (!rawConfig.providers || rawConfig.providers.length === 0) { + logger.warn('Configuration file contains no providers. Nothing to migrate.'); + return false; + } + + // 1. Process and install plugins + const processedPlugins = new Set(); + const failedPlugins = new Set(); + + for (const entry of rawConfig.providers) { + const packageName = entry.package; + if (processedPlugins.has(packageName) || failedPlugins.has(packageName)) { + continue; + } + + const targetVersion = entry.version || 'latest'; + const installedPath = join(PLUGINS_NODE_MODULES, packageName); + + // If not already in node_modules, install it + let installFailed = false; + if (!existsSync(installedPath)) { + logger.info(`Migrating plugin: Installing '${packageName}@${targetVersion}'...`); + try { + installNpmPackage(packageName, targetVersion); + } catch (installErr) { + logger.error(`Could not install plugin package '${packageName}':`, installErr); + installFailed = true; + } + } + + if (installFailed) { + failedPlugins.add(packageName); + continue; + } + + try { + const { manifest, version: authoritativeVersion } = await extractPackageManifest(packageName); + if (!manifest?.channel) { + throw new Error(`Package '${packageName}' manifest does not specify a valid channel.`); + } + + await Plugin.findOneAndUpdate( + { name: packageName }, + { + name: packageName, + version: authoritativeVersion, + status: 'installed', + manifest, + error: undefined, + }, + { upsert: true, new: true } + ); + processedPlugins.add(packageName); + logger.success(`Migrated plugin '${packageName}' (version: ${authoritativeVersion})`); + } catch (manifestErr) { + logger.error(`Failed to extract manifest for '${packageName}':`, manifestErr); + failedPlugins.add(packageName); + } + } + + // 2. Process and encrypt provider configurations + const migratedProviderIds = new Set(); + + for (const entry of rawConfig.providers) { + // Find channel from installed plugin manifest + const pluginDoc = await Plugin.findOne({ name: entry.package, status: 'installed' }); + const channel = pluginDoc?.manifest?.channel; + + if (!pluginDoc || !channel) { + logger.error( + `Skipping migration for provider '${entry.id}': Plugin '${entry.package}' is not installed or has no valid channel in manifest.` + ); + continue; + } + + try { + const resolvedCreds = resolveCredentials(entry.credentials); + const resolvedOpts = resolveOptionalConfig(entry.optionalConfig); + const mergedOptions = { + ...(entry.options || {}), + ...resolvedOpts, + }; + + const encryptedCreds = await encryptCredentials(resolvedCreds); + + await Provider.findOneAndUpdate( + { id: entry.id }, + { + id: entry.id, + plugin_name: entry.package, + channel, + enabled: true, + credentials: encryptedCreds, + options: mergedOptions, + }, + { upsert: true, new: true } + ); + + migratedProviderIds.add(entry.id); + logger.success(`Migrated provider '${entry.id}' (${channel})`); + } catch (providerErr) { + logger.error(`Failed to migrate provider '${entry.id}':`, providerErr); + } + } + + if (migratedProviderIds.size === 0) { + logger.warn('No providers were successfully migrated from YAML configuration.'); + return false; + } + + // 3. Process channel routing + if (rawConfig.channels) { + for (const [channel, route] of Object.entries(rawConfig.channels)) { + let fallbackArray: string[] = []; + if (Array.isArray(route.fallback)) { + fallbackArray = route.fallback; + } else if (typeof route.fallback === 'string' && route.fallback.trim().length > 0) { + fallbackArray = [route.fallback.trim()]; + } + + // Filter fallback providers to those actually migrated and matching this channel + const validFallbacks: string[] = []; + for (const fId of fallbackArray) { + if (migratedProviderIds.has(fId)) { + validFallbacks.push(fId); + } else { + const exists = await Provider.findOne({ id: fId, channel }); + if (exists) { + validFallbacks.push(fId); + } else { + logger.warn( + `Omitting fallback provider '${fId}' from channel routing '${channel}': provider not migrated or does not exist for this channel.` + ); + } + } + } + + // Validate default provider + let effectiveDefault = (migratedProviderIds.has(route.default) || (await Provider.findOne({ id: route.default, channel }))) + ? route.default + : undefined; + + if (!effectiveDefault) { + if (validFallbacks.length > 0) { + effectiveDefault = validFallbacks.shift()!; + logger.warn( + `Default provider '${route.default}' for channel '${channel}' was not migrated. Promoted fallback '${effectiveDefault}' as default.` + ); + } else { + logger.error( + `Skipping channel routing for '${channel}': default provider '${route.default}' was not migrated and no valid fallbacks are available.` + ); + continue; + } + } + + await ChannelRouting.findOneAndUpdate( + { channel }, + { + channel, + default_provider_id: effectiveDefault, + fallback_provider_ids: validFallbacks, + partitions: route.partitions || 6, + }, + { upsert: true, new: true } + ); + + logger.success( + `Migrated channel routing '${channel}': default=${effectiveDefault}, fallbacks=[${validFallbacks.join(', ')}]` + ); + } + } + + logger.success('YAML configuration migration completed successfully.'); + + // Broadcast SYSTEM_RELOAD event to notify other instances + await PluginSyncService.publish('SYSTEM_RELOAD', {}); + return true; + } +} diff --git a/src/plugins/crypto/keypair-manager.ts b/src/plugins/crypto/keypair-manager.ts new file mode 100644 index 0000000..ffe62eb --- /dev/null +++ b/src/plugins/crypto/keypair-manager.ts @@ -0,0 +1,173 @@ +/** + * Key-Pair Envelope Encryption Manager + * + * Secures provider credentials using RSA-OAEP (2048-bit) + AES-256-GCM envelope encryption. + * Automatically generates, caches, and persists RSA key pair in MongoDB or loads from environment. + */ + +import crypto from 'crypto'; +import system_config_model from '@src/database/models/system-config.models.js'; +import type { encrypted_credentials } from '@src/types/types.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +const PUBLIC_KEY_CONFIG = 'plugin_credentials_public_key'; +const PRIVATE_KEY_CONFIG = 'plugin_credentials_private_key'; + +let cachedPublicKey: string | null = null; +let cachedPrivateKey: string | null = null; + +/** + * Generate a new 2048-bit RSA key pair in PEM format + */ +export function generateRSAKeyPair(): { publicKey: string; privateKey: string } { + return crypto.generateKeyPairSync('rsa', { + modulusLength: 2048, + publicKeyEncoding: { + type: 'spki', + format: 'pem', + }, + privateKeyEncoding: { + type: 'pkcs8', + format: 'pem', + }, + }); +} + +/** + * Get or create RSA Public and Private Key Pair + * Priority: Environment Variables > MongoDB SystemConfig > Generate & Persist + */ +export async function getOrCreateKeyPair(): Promise<{ publicKey: string; privateKey: string }> { + if (cachedPublicKey && cachedPrivateKey) { + return { publicKey: cachedPublicKey, privateKey: cachedPrivateKey }; + } + + // 1. Check environment variables + const envPub = process.env.SIMPLENS_PLUGIN_PUBLIC_KEY; + const envPriv = process.env.SIMPLENS_PLUGIN_PRIVATE_KEY; + if (envPub && envPriv) { + cachedPublicKey = envPub.replace(/\\n/g, '\n'); + cachedPrivateKey = envPriv.replace(/\\n/g, '\n'); + return { publicKey: cachedPublicKey, privateKey: cachedPrivateKey }; + } + + // 2. Check MongoDB for existing keys + const [pubDoc, privDoc] = await Promise.all([ + system_config_model.findOne({ key: PUBLIC_KEY_CONFIG }), + system_config_model.findOne({ key: PRIVATE_KEY_CONFIG }), + ]); + + if (pubDoc?.value && privDoc?.value) { + cachedPublicKey = pubDoc.value as string; + cachedPrivateKey = privDoc.value as string; + return { publicKey: cachedPublicKey, privateKey: cachedPrivateKey }; + } + + // 3. Generate new key pair and store in MongoDB + logger.info('Generating new RSA 2048-bit key pair for provider credential encryption...'); + const keyPair = generateRSAKeyPair(); + + await Promise.all([ + system_config_model.findOneAndUpdate( + { key: PUBLIC_KEY_CONFIG }, + { value: keyPair.publicKey }, + { upsert: true, new: true } + ), + system_config_model.findOneAndUpdate( + { key: PRIVATE_KEY_CONFIG }, + { value: keyPair.privateKey }, + { upsert: true, new: true } + ), + ]); + + cachedPublicKey = keyPair.publicKey; + cachedPrivateKey = keyPair.privateKey; + logger.success('RSA key pair generated and persisted to MongoDB system_configs'); + + return { publicKey: cachedPublicKey, privateKey: cachedPrivateKey }; +} + +/** + * Encrypt credentials using hybrid envelope encryption (AES-256-GCM + RSA-OAEP) + * @param plaintext - Credentials object or JSON string to encrypt + */ +export async function encryptCredentials( + plaintext: Record | string +): Promise { + const { publicKey } = await getOrCreateKeyPair(); + const dataString = typeof plaintext === 'string' ? plaintext : JSON.stringify(plaintext); + + // 1. Generate random 256-bit AES Data Encryption Key (DEK) + const dek = crypto.randomBytes(32); + const iv = crypto.randomBytes(16); + + // 2. Encrypt plaintext credentials with AES-256-GCM + const cipher = crypto.createCipheriv('aes-256-gcm', dek, iv, { + authTagLength: 16, + }); + + let encryptedData = cipher.update(dataString, 'utf8', 'hex'); + encryptedData += cipher.final('hex'); + const authTag = cipher.getAuthTag().toString('hex'); + + // 3. Encrypt AES DEK with RSA Public Key (RSA-OAEP with SHA-256) + const encryptedDek = crypto.publicEncrypt( + { + key: publicKey, + padding: crypto.constants.RSA_PKCS1_OAEP_PADDING, + oaepHash: 'sha256', + }, + dek + ).toString('hex'); + + return { + encrypted_data: encryptedData, + iv: iv.toString('hex'), + auth_tag: authTag, + encrypted_dek: encryptedDek, + }; +} + +/** + * Decrypt credentials using hybrid envelope encryption (AES-256-GCM + RSA-OAEP) + * @param credentials - Encrypted credentials subdocument + * @returns Decrypted Record + */ +export async function decryptCredentials( + credentials: encrypted_credentials +): Promise> { + const { privateKey } = await getOrCreateKeyPair(); + + // 1. Decrypt AES DEK using RSA Private Key + const dek = crypto.privateDecrypt( + { + key: privateKey, + padding: crypto.constants.RSA_PKCS1_OAEP_PADDING, + oaepHash: 'sha256', + }, + Buffer.from(credentials.encrypted_dek, 'hex') + ); + + // 2. Decrypt ciphertext using AES-256-GCM + const decipher = crypto.createDecipheriv( + 'aes-256-gcm', + dek, + Buffer.from(credentials.iv, 'hex'), + { authTagLength: 16 } + ); + + decipher.setAuthTag(Buffer.from(credentials.auth_tag, 'hex')); + + let decrypted = decipher.update(credentials.encrypted_data, 'hex', 'utf8'); + decrypted += decipher.final('utf8'); + + return JSON.parse(decrypted) as Record; +} + +/** + * Clear cached key pair (used in tests) + */ +export function clearKeyPairCache(): void { + cachedPublicKey = null; + cachedPrivateKey = null; +} diff --git a/src/plugins/index.ts b/src/plugins/index.ts index da6e5e6..85be2a0 100644 --- a/src/plugins/index.ts +++ b/src/plugins/index.ts @@ -16,7 +16,13 @@ export { } from './loader/registry.js'; // Loader -export { loadProviders, loadProvidersFromEnv, registerProvider, getConfiguredChannels } from './loader/loader.js'; +export { + loadProviders, + loadProvidersFromEnv, + loadProvidersFromDatabase, + getConfiguredChannels, + registerProvider, +} from './loader/loader.js'; // Router export { @@ -24,6 +30,23 @@ export { sendToProvider, validateNotification, validateNotificationForProvider, + getProviderCascade, resolveFallbackProviderId, getRateLimitConfig } from './loader/router.js'; + +// Dynamic Management Services +export { PluginManagerService } from './services/plugin-manager.service.js'; +export { NpmAuthService } from './services/npm-auth.service.js'; +export { ProviderManagerService, type ProviderResponseDto } from './services/provider-manager.service.js'; +export { ChannelRoutingService } from './services/channel-routing.service.js'; +export { YamlMigrator } from './bootstrap/yaml-migrator.js'; +export { PluginSyncService } from './sync/plugin-sync.service.js'; +export { encryptCredentials, decryptCredentials } from './crypto/keypair-manager.js'; +export { + installNpmPackage, + uninstallNpmPackage, + extractPackageManifest, + importAndInstantiateProvider, + validateSimpleNSProvider, +} from './loader/plugin-fs.js'; diff --git a/src/plugins/interfaces/provider.types.ts b/src/plugins/interfaces/provider.types.ts deleted file mode 100644 index 50c0cfe..0000000 --- a/src/plugins/interfaces/provider.types.ts +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Provider Types - * - * Type definitions for SimpleNS provider plugins. - * These mirror the @simplens/sdk interfaces for use within the core. - */ - -import { z } from 'zod'; - -/** - * Delivery result from sending a notification - */ -export interface DeliveryResult { - success: boolean; - messageId?: string; - providerResponse?: unknown; - error?: { - code: string; - message: string; - retryable: boolean; - }; -} - -/** - * Provider configuration passed during initialization - */ -export interface ProviderConfig { - id: string; - credentials: Record; - options?: Record; -} - -/** - * Rate limit configuration - */ -export interface RateLimitConfig { - maxTokens: number; - refillRate: number; - refillInterval?: 'second' | 'minute' | 'hour' | 'day'; -} - -/** - * Provider manifest - metadata about the provider - */ -export interface ProviderManifest { - name: string; - version: string; - channel: string; - displayName: string; - description: string; - author: string; - homepage?: string; - requiredCredentials: string[]; - optionalConfig?: string[]; -} - -/** - * Base notification fields common to all channels - */ -export interface BaseNotification { - notification_id: string; - request_id: string; - client_id: string; - channel: string; - provider?: string; - recipient: Record; - content: Record; - variables?: Record; - webhook_url: string; - retry_count: number; - created_at: Date; -} - -/** - * SimpleNS Provider Interface - */ -export interface SimpleNSProvider { - readonly manifest: ProviderManifest; - - getNotificationSchema(): z.ZodSchema; - getRecipientSchema(): z.ZodObject>; - getContentSchema(): z.ZodObject>; - getRateLimitConfig(): RateLimitConfig; - - initialize(config: ProviderConfig): Promise; - healthCheck(): Promise; - send(notification: TNotification): Promise; - shutdown(): Promise; -} diff --git a/src/plugins/loader/loader.ts b/src/plugins/loader/loader.ts index 3d9d67d..8469837 100644 --- a/src/plugins/loader/loader.ts +++ b/src/plugins/loader/loader.ts @@ -5,40 +5,31 @@ * Supports dynamic import of npm packages. */ -import { readFileSync, existsSync, mkdirSync, writeFileSync } from 'fs'; +import { readFileSync, existsSync } from 'fs'; import { join } from 'path'; -import { execSync } from 'child_process'; -import { pathToFileURL } from 'url'; import { parse as parseYaml } from 'yaml'; import { PluginRegistry, type ChannelConfig } from './registry.js'; -import type { SimpleNSProvider, ProviderConfig } from '../interfaces/provider.types.js'; import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; - -// Plugins directory for user-installed plugins -const PLUGINS_DIR = join(process.cwd(), '.plugins'); -const PLUGINS_NODE_MODULES = join(PLUGINS_DIR, 'node_modules'); - -/** - * Initialize plugins directory with package.json if needed - */ -function initPluginsDir(): void { - if (!existsSync(PLUGINS_DIR)) { - mkdirSync(PLUGINS_DIR, { recursive: true }); - } - - const packageJsonPath = join(PLUGINS_DIR, 'package.json'); - if (!existsSync(packageJsonPath)) { - const initialPackage = { - name: 'simplens-plugins', - version: '1.0.0', - description: 'User-installed SimpleNS plugins', - private: true, - type: 'module', - dependencies: {} - }; - writeFileSync(packageJsonPath, JSON.stringify(initialPackage, null, 2)); - } -} +import Plugin from '@src/database/models/plugin.models.js'; +import Provider from '@src/database/models/provider.models.js'; +import ChannelRouting from '@src/database/models/channel-routing.models.js'; +import { ProviderManagerService } from '@src/plugins/services/provider-manager.service.js'; +import { + PLUGINS_NODE_MODULES, + installNpmPackage, + importAndInstantiateProvider, + resolveCredentials, + resolveOptionalConfig, + findConfigFile, +} from './plugin-fs.js'; +import type { + provider_document, + ProviderEntry, + ProviderOptions, + SimpleNSConfig, + SimpleNSProvider, + ProviderConfig, +} from '@src/types/types.js'; /** * Install missing plugins from configuration @@ -63,94 +54,18 @@ async function installMissingPlugins(config: SimpleNSConfig): Promise { } logger.info(`Auto-installing ${missingPackages.length} missing plugin(s)...`); - initPluginsDir(); for (const pkg of missingPackages) { - logger.info(`Installing ${pkg}...`); - try { - execSync(`npm install ${pkg}`, { - cwd: PLUGINS_DIR, - stdio: 'pipe' // Suppress output for cleaner logs - }); - logger.success(`Installed ${pkg}`); - } catch (err) { - logger.error(`Failed to install ${pkg}`, err); - const wrappedErr = new Error(`Failed to auto-install plugin: ${pkg}`); - (wrappedErr as unknown as Record).cause = err; - throw wrappedErr; - } + installNpmPackage(pkg); } } -/** - * Provider entry in configuration - */ -interface ProviderEntry { - /** npm package name (e.g., '@simplens/gmail') */ - package: string; - /** Unique ID for this provider instance */ - id: string; - /** Provider credentials */ - credentials: Record; - /** Optional config values (resolved from env vars) */ - optionalConfig?: Record; - /** Optional settings */ - options?: { - priority?: number; - rateLimit?: { - maxTokens?: number; - refillRate?: number; - }; - [key: string]: unknown; - }; -} - -/** - * SimpleNS configuration file structure - */ -interface SimpleNSConfig { - /** Provider configurations */ - providers: ProviderEntry[]; - /** Channel to provider mapping */ - channels?: Record; -} - -/** - * Find local config file if it exists - * Checks for simplens.config.yaml/.yml/.json - */ -function findLocalConfig(basePath: string): string | null { - const dir = basePath.substring(0, basePath.lastIndexOf('/') + 1) || './'; - const localPatterns = [ - 'simplens.config.yaml', - 'simplens.config.yml', - 'simplens.config.json' - ]; - - for (const filename of localPatterns) { - const localPath = dir + filename; - if (existsSync(localPath)) { - return localPath; - } - } - return null; -} - /** * Load configuration from file * Returns null if no config file exists */ function loadConfig(configPath: string): SimpleNSConfig | null { - // Check for local config first (takes precedence) - const localConfigPath = findLocalConfig(configPath); - const pathToUse = localConfigPath || configPath; - - if (localConfigPath) { - logger.info(`Using local config: ${localConfigPath}`); - } + const pathToUse = findConfigFile(configPath) || configPath; if (!existsSync(pathToUse)) { // No config file - this is okay, just means no plugins configured @@ -169,144 +84,6 @@ function loadConfig(configPath: string): SimpleNSConfig | null { } } -/** - * Resolve environment variables in credentials - * Supports ${VAR_NAME} syntax - */ -function resolveCredentials(credentials: Record | undefined): Record { - const resolved: Record = {}; - - for (const [key, value] of Object.entries(credentials || {})) { - if (typeof value === 'string' && value.startsWith('${') && value.endsWith('}')) { - const envVar = value.slice(2, -1); - const envValue = process.env[envVar]; - if (!envValue) { - logger.warn(`Environment variable ${envVar} not set`); - } - resolved[key] = envValue || ''; - } else { - resolved[key] = value; - } - } - - return resolved; -} - -/** - * Resolve optional config from environment variables - * Takes the optionalConfig from the config entry and resolves ${VAR} placeholders - * If env var is not set, the key is removed (no error since it's optional) - */ -function resolveOptionalConfig( - optionalConfig: Record | undefined -): Record { - const resolved: Record = {}; - - if (!optionalConfig) return resolved; - - for (const [key, value] of Object.entries(optionalConfig)) { - // Check if value is a placeholder like ${VAR_NAME} - if (typeof value === 'string' && value.startsWith('${') && value.endsWith('}')) { - const envVar = value.slice(2, -1); - const envValue = process.env[envVar]; - - if (envValue) { - resolved[key] = envValue; - logger.debug(`Loaded optional config: ${key}`); - } - // If not set, just skip (don't add to resolved) - } else { - // Literal value, keep as-is - resolved[key] = value; - } - } - - return resolved; -} - -/** - * Helper to instantiate provider from module - */ -function instantiateFromModule(module: Record, packageName: string): SimpleNSProvider { - if (module.default) { - if (typeof module.default === 'function') { - const DefaultExport = module.default as new () => SimpleNSProvider; - if (DefaultExport.prototype?.constructor) { - return new DefaultExport(); - } - // Factory function - return (module.default as () => SimpleNSProvider)(); - } - return module.default as SimpleNSProvider; - } - if (module.createProvider) { - return (module.createProvider as () => SimpleNSProvider)(); - } - throw new Error(`Package ${packageName} does not export a provider`); -} - -/** - * Resolve the entry file for a package from its package.json - */ -function resolvePackageEntry(packagePath: string): string { - const pkgJsonPath = join(packagePath, 'package.json'); - if (!existsSync(pkgJsonPath)) { - return join(packagePath, 'index.js'); // Fallback - } - - const pkg = JSON.parse(readFileSync(pkgJsonPath, 'utf-8')); - - // Resolve entry point: exports > main > index.js - let entryPoint = 'index.js'; - if (pkg.exports) { - if (typeof pkg.exports === 'string') { - entryPoint = pkg.exports; - } else if (pkg.exports['.']) { - const dotExport = pkg.exports['.']; - entryPoint = typeof dotExport === 'string' - ? dotExport - : (dotExport.import || dotExport.default || 'index.js'); - } - } else if (pkg.main) { - entryPoint = pkg.main; - } - - return join(packagePath, entryPoint); -} - -/** - * Dynamically import a provider package and instantiate - * First checks plugins directory, then falls back to core node_modules - */ -async function importAndInstantiateProvider(packageName: string): Promise { - try { - // First, try to import from plugins directory - const pluginPath = join(PLUGINS_NODE_MODULES, packageName); - if (existsSync(pluginPath)) { - logger.debug(`Loading from plugins directory: ${packageName}`); - // Resolve the actual entry file from package.json - const entryFile = resolvePackageEntry(pluginPath); - // Convert to file:// URL for cross-platform compatibility - const pluginUrl = pathToFileURL(entryFile).href; - const module = await import(pluginUrl) as Record; - return instantiateFromModule(module, packageName); - } - - // Fall back to core node_modules (for bundled plugins or testing) - const module = await import(packageName) as Record; - return instantiateFromModule(module, packageName); - } catch (_err) { - // Handle local path imports - if (packageName.startsWith('./') || packageName.startsWith('../')) { - const module = await import(packageName) as Record; - return instantiateFromModule(module, packageName); - } - const wrappedErr = new Error(`Plugin not found: ${packageName}. Install with: npm run plugin:install ${packageName}`); - (wrappedErr as unknown as Record).cause = _err; - throw wrappedErr; - } -} - /** * Load a single provider */ @@ -431,7 +208,7 @@ export async function registerProvider( provider: SimpleNSProvider, id: string, credentials: Record, - options?: Record + options?: ProviderOptions ): Promise { await provider.initialize({ id, @@ -439,6 +216,69 @@ export async function registerProvider( options, }); - const priority = (options?.priority as number) || 0; + const priority = options?.priority ?? 0; PluginRegistry.register(provider, id, priority); } + +/** + * Load all providers and channel routings directly from MongoDB + */ +export async function loadProvidersFromDatabase(options: { initialize?: boolean } = {}): Promise { + const shouldInitialize = options.initialize !== false; + logger.info('Loading provider plugins from MongoDB...'); + + // 1. Ensure all installed plugins exist locally + try { + const installedPlugins = await Plugin.find({ status: 'installed' }).lean(); + for (const plugin of installedPlugins) { + const pluginPath = join(PLUGINS_NODE_MODULES, plugin.name); + if (!existsSync(pluginPath)) { + logger.info(`Syncing missing plugin package locally: ${plugin.name}@${plugin.version}`); + try { + installNpmPackage(plugin.name, plugin.version); + } catch (err) { + logger.error(`Failed to install missing package ${plugin.name}:`, err); + } + } + } + } catch (dbErr) { + logger.error('Error fetching installed plugins from database:', dbErr); + } + + // 2. Load all enabled providers + try { + const providers = await Provider.find({ enabled: true }).lean(); + logger.info(`Found ${providers.length} enabled provider(s) in MongoDB.`); + + for (const providerDoc of providers) { + try { + await ProviderManagerService.loadAndRegisterProvider( + providerDoc as unknown as provider_document, + shouldInitialize + ); + } catch (err) { + logger.error(`Failed to load provider '${providerDoc.id}':`, err); + } + } + } catch (dbErr) { + logger.error('Error fetching providers from database:', dbErr); + } + + // 3. Load channel routing + try { + const routings = await ChannelRouting.find().lean(); + for (const route of routings) { + PluginRegistry.setChannelConfig(route.channel, { + default: route.default_provider_id, + fallback: route.fallback_provider_ids || [], + }); + } + } catch (dbErr) { + logger.error('Error fetching channel routings from database:', dbErr); + } + + PluginRegistry.setInitialized(true); + logger.success(`Loaded ${PluginRegistry.getProviderIds().length} providers from MongoDB.`); + logger.info(`Configured channels: ${PluginRegistry.getChannels().join(', ')}`); +} + diff --git a/src/plugins/loader/plugin-fs.ts b/src/plugins/loader/plugin-fs.ts new file mode 100644 index 0000000..3f5a9e8 --- /dev/null +++ b/src/plugins/loader/plugin-fs.ts @@ -0,0 +1,355 @@ +/** + * Plugin File System & Package Utilities + * + * Manages local .plugins directory, dynamic imports, manifest extraction, + * and npm package installation/uninstallation. + */ + +import { readFileSync, existsSync, mkdirSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { spawnSync } from 'child_process'; +import { pathToFileURL } from 'url'; +import type { SimpleNSProvider, ProviderManifest } from '@src/types/types.js'; +import { + providerManifestSchema, + npmPackageNameSchema, + npmPackageVersionSchema, +} from '@src/types/schemas.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +export const PLUGINS_DIR = join(process.cwd(), '.plugins'); +export const PLUGINS_NODE_MODULES = join(PLUGINS_DIR, 'node_modules'); + +/** + * Resolve environment variables in credentials (${VAR_NAME} syntax) + */ +export function resolveCredentials(credentials: Record | undefined): Record { + const resolved: Record = {}; + + for (const [key, value] of Object.entries(credentials || {})) { + if (typeof value === 'string' && value.startsWith('${') && value.endsWith('}')) { + const envVar = value.slice(2, -1); + const envValue = process.env[envVar]; + if (!envValue) { + logger.warn(`Environment variable ${envVar} not set`); + } + resolved[key] = envValue || ''; + } else { + resolved[key] = value; + } + } + + return resolved; +} + +/** + * Resolve optional config from environment variables (${VAR_NAME} syntax) + */ +export function resolveOptionalConfig( + optionalConfig: Record | undefined +): Record { + const resolved: Record = {}; + if (!optionalConfig) return resolved; + + for (const [key, value] of Object.entries(optionalConfig)) { + if (typeof value === 'string' && value.startsWith('${') && value.endsWith('}')) { + const envVar = value.slice(2, -1); + const envValue = process.env[envVar]; + if (envValue) { + resolved[key] = envValue; + logger.debug(`Loaded optional config: ${key}`); + } + } else { + resolved[key] = value; + } + } + + return resolved; +} + +/** + * Find local config file if it exists + */ +export function findConfigFile(basePath?: string): string | null { + const customPath = process.env.SIMPLENS_CONFIG_PATH; + if (customPath && existsSync(customPath)) { + return customPath; + } + + const dir = basePath ? basePath.substring(0, basePath.lastIndexOf('/') + 1) || './' : './'; + const candidates = [ + dir + 'simplens.config.yaml', + dir + 'simplens.config.yml', + dir + 'simplens.config.json', + join(process.cwd(), 'simplens.config.yaml'), + join(process.cwd(), 'simplens.config.yml'), + join(process.cwd(), 'simplens.config.json'), + ]; + + for (const p of candidates) { + if (existsSync(p)) return p; + } + return null; +} + +/** + * Initialize plugins directory with package.json if needed + */ +export function initPluginsDir(): void { + if (!existsSync(PLUGINS_DIR)) { + mkdirSync(PLUGINS_DIR, { recursive: true }); + } + + const packageJsonPath = join(PLUGINS_DIR, 'package.json'); + if (!existsSync(packageJsonPath)) { + const initialPackage = { + name: 'simplens-plugins', + version: '1.0.0', + description: 'User-installed SimpleNS plugins', + private: true, + type: 'module', + dependencies: {}, + }; + writeFileSync(packageJsonPath, JSON.stringify(initialPackage, null, 2)); + } +} + +/** + * Resolve the entry file for a package from its package.json + */ +export function resolvePackageEntry(packagePath: string): string { + const pkgJsonPath = join(packagePath, 'package.json'); + if (!existsSync(pkgJsonPath)) { + return join(packagePath, 'index.js'); + } + + const pkg = JSON.parse(readFileSync(pkgJsonPath, 'utf-8')); + + let entryPoint = 'index.js'; + if (pkg.exports) { + if (typeof pkg.exports === 'string') { + entryPoint = pkg.exports; + } else if (pkg.exports['.']) { + const dotExport = pkg.exports['.']; + entryPoint = + typeof dotExport === 'string' + ? dotExport + : dotExport.import || dotExport.default || 'index.js'; + } + } else if (pkg.main) { + entryPoint = pkg.main; + } + + return join(packagePath, entryPoint); +} + +/** + * Helper to instantiate provider from imported module + */ +export function instantiateFromModule( + module: Record, + packageName: string +): SimpleNSProvider { + if (module.default) { + if (typeof module.default === 'function') { + const DefaultExport = module.default as new () => SimpleNSProvider; + if (DefaultExport.prototype?.constructor) { + return new DefaultExport(); + } + return (module.default as () => SimpleNSProvider)(); + } + return module.default as SimpleNSProvider; + } + if (module.createProvider) { + return (module.createProvider as () => SimpleNSProvider)(); + } + throw new Error(`Package ${packageName} does not export a provider`); +} + +/** + * Dynamically import a provider package and instantiate + */ +export async function importAndInstantiateProvider(packageName: string): Promise { + try { + const pluginPath = join(PLUGINS_NODE_MODULES, packageName); + if (existsSync(pluginPath)) { + logger.debug(`Loading from plugins directory: ${packageName}`); + const entryFile = resolvePackageEntry(pluginPath); + const pluginUrl = pathToFileURL(entryFile).href; + const module = (await import(pluginUrl)) as Record; + return instantiateFromModule(module, packageName); + } + + // Fall back to core node_modules (for bundled plugins or testing) + const module = (await import(packageName)) as Record; + return instantiateFromModule(module, packageName); + } catch (_err) { + if (packageName.startsWith('./') || packageName.startsWith('../')) { + const module = (await import(packageName)) as Record; + return instantiateFromModule(module, packageName); + } + const wrappedErr = new Error(`Plugin not found: ${packageName}. Install with npm or dashboard.`); + (wrappedErr as unknown as Record).cause = _err; + throw wrappedErr; + } +} + +/** + * Validate that an instantiated object strictly satisfies the SimpleNSProvider contract + */ +export function validateSimpleNSProvider( + provider: unknown, + packageName: string +): SimpleNSProvider { + if (!provider || typeof provider !== 'object') { + throw new Error( + `Package '${packageName}' is not a valid SimpleNS plugin: default export must be a Provider object or class.` + ); + } + + const p = provider as Record; + + if (!p.manifest || typeof p.manifest !== 'object') { + throw new Error( + `Package '${packageName}' is not a valid SimpleNS plugin: missing 'manifest' definition.` + ); + } + + const manifestValidation = providerManifestSchema.safeParse(p.manifest); + if (!manifestValidation.success) { + const errorDetails = Object.entries(manifestValidation.error.flatten().fieldErrors) + .map(([k, v]) => `${k}: ${v?.join(', ')}`) + .join('; '); + throw new Error( + `Package '${packageName}' has an invalid plugin manifest: ${errorDetails}` + ); + } + + const requiredMethods: (keyof SimpleNSProvider)[] = [ + 'send', + 'initialize', + 'healthCheck', + 'shutdown', + 'getNotificationSchema', + 'getRecipientSchema', + 'getContentSchema', + 'getRateLimitConfig', + ]; + + const missingMethods = requiredMethods.filter((method) => typeof p[method] !== 'function'); + + if (missingMethods.length > 0) { + throw new Error( + `Package '${packageName}' is not a valid SimpleNS plugin. Missing required provider methods: ${missingMethods.join(', ')}` + ); + } + + return provider as SimpleNSProvider; +} + +/** + * Extract manifest and authoritative package.json version from an installed package + */ +export async function extractPackageManifest( + packageName: string +): Promise<{ manifest: ProviderManifest; version: string }> { + let packageDir = join(PLUGINS_NODE_MODULES, packageName); + if (!existsSync(packageDir)) { + // Fall back to root node_modules if present + packageDir = join(process.cwd(), 'node_modules', packageName); + } + + const pkgJsonPath = join(packageDir, 'package.json'); + if (!existsSync(pkgJsonPath)) { + throw new Error(`package.json not found for installed plugin '${packageName}'`); + } + + const pkgJson = JSON.parse(readFileSync(pkgJsonPath, 'utf-8')); + const version: string = pkgJson.version || '1.0.0'; + + const rawProvider = await importAndInstantiateProvider(packageName); + const provider = validateSimpleNSProvider(rawProvider, packageName); + + // Ensure manifest version matches package.json + const manifest: ProviderManifest = { + ...provider.manifest, + version, + }; + + return { manifest, version }; +} + +/** + * Install an npm package into .plugins + */ +export function installNpmPackage(packageName: string, version?: string): void { + const pkgValidation = npmPackageNameSchema.safeParse(packageName); + if (!pkgValidation.success) { + throw new Error(`Invalid npm package name '${packageName}'`); + } + if (version && version !== 'latest') { + const verValidation = npmPackageVersionSchema.safeParse(version); + if (!verValidation.success) { + throw new Error(`Invalid npm package version '${version}'`); + } + } + + initPluginsDir(); + const specifier = version && version !== 'latest' ? `${packageName}@${version}` : `${packageName}@latest`; + logger.info(`Running npm install ${specifier} in ${PLUGINS_DIR}...`); + + const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm'; + const result = spawnSync(npmCmd, ['install', specifier], { + cwd: PLUGINS_DIR, + stdio: 'pipe', + shell: process.platform === 'win32', + }); + + if (result.status !== 0 || result.error) { + const stderrMsg = result.stderr ? result.stderr.toString('utf-8') : (result.error?.message || ''); + logger.error(`Failed to install ${specifier}:`, stderrMsg || result.error); + + let friendlyMessage = `Failed to install package '${specifier}'`; + if (stderrMsg.includes('E401') || stderrMsg.includes('401 Unauthorized')) { + friendlyMessage = `npm authentication failed (401 Unauthorized) for package '${packageName}'. Verify your npm token.`; + } else if (stderrMsg.includes('E404') || stderrMsg.includes('404 Not Found')) { + friendlyMessage = `Package '${packageName}' was not found (404 Not Found) on npm registry.`; + } else if (stderrMsg.includes('ETARGET')) { + friendlyMessage = `Version '${version}' not found for package '${packageName}'.`; + } else if (stderrMsg.trim()) { + friendlyMessage += `: ${stderrMsg.trim().slice(0, 300)}`; + } + + throw new Error(friendlyMessage, { cause: result.error }); + } + + logger.success(`Successfully installed ${specifier}`); +} + +/** + * Uninstall an npm package from .plugins + */ +export function uninstallNpmPackage(packageName: string): void { + const pkgValidation = npmPackageNameSchema.safeParse(packageName); + if (!pkgValidation.success) { + throw new Error(`Invalid npm package name '${packageName}'`); + } + + initPluginsDir(); + logger.info(`Running npm uninstall ${packageName} in ${PLUGINS_DIR}...`); + + const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm'; + const result = spawnSync(npmCmd, ['uninstall', packageName], { + cwd: PLUGINS_DIR, + stdio: 'pipe', + shell: process.platform === 'win32', + }); + + if (result.status !== 0 || result.error) { + const stderrMsg = result.stderr ? result.stderr.toString('utf-8') : (result.error?.message || ''); + logger.error(`Failed to uninstall ${packageName}:`, stderrMsg || result.error); + throw new Error(`Failed to uninstall package: ${packageName}`, { cause: result.error }); + } + + logger.success(`Successfully uninstalled ${packageName}`); +} diff --git a/src/plugins/loader/registry.ts b/src/plugins/loader/registry.ts index e056f71..e3741bd 100644 --- a/src/plugins/loader/registry.ts +++ b/src/plugins/loader/registry.ts @@ -5,65 +5,21 @@ * Handles provider lookup by ID or channel. */ -import type { SimpleNSProvider, ProviderManifest } from '../interfaces/provider.types.js'; +import type { SimpleNSProvider, ProviderManifest, RegisteredProvider, ChannelConfig, FieldDefinition, ProviderMetadata, ChannelMetadata, PluginMetadata } from '@src/types/types.js'; import { pluginRegistryLogger as logger } from '@src/workers/utils/logger.js'; -/** - * Registered provider with configuration - */ -export interface RegisteredProvider { - provider: SimpleNSProvider; - id: string; - priority: number; -} +export type { + RegisteredProvider, + ChannelConfig, + FieldDefinition, + ProviderMetadata, + ChannelMetadata, + PluginMetadata, +} from '@src/types/types.js'; /** - * Channel configuration - */ -export interface ChannelConfig { - default: string; - fallback?: string; -} - -/** - * Field definition for dynamic form generation - */ -export interface FieldDefinition { - name: string; - type: 'string' | 'email' | 'phone' | 'text' | 'number' | 'boolean'; - required: boolean; - description?: string; -} - -/** - * Provider metadata for dashboard - */ -export interface ProviderMetadata { - id: string; - name: string; - displayName: string; - description?: string; - priority: number; - recipientFields: FieldDefinition[]; - contentFields: FieldDefinition[]; -} - -/** - * Channel metadata for dashboard + * Registered provider with configuration */ -export interface ChannelMetadata { - providers: ProviderMetadata[]; - default?: string; - fallback?: string; -} - -/** - * Full plugin metadata response - */ -export interface PluginMetadata { - channels: Record; -} - /** * Extract field definitions from a Zod object schema */ @@ -156,6 +112,73 @@ class PluginRegistryClass { logger.info(`Registered provider: ${id} (${provider.manifest.displayName}) for channel: ${channel}`); } + /** + * Register or replace an existing provider + */ + registerOrReplace(provider: SimpleNSProvider, id: string, priority: number = 0): void { + if (this.providers.has(id)) { + const registered = this.providers.get(id); + const channel = registered?.provider.manifest.channel; + const savedConfig = channel ? this.channelConfig.get(channel) : undefined; + const wasDefault = savedConfig?.default === id; + const fallbackIndex = Array.isArray(savedConfig?.fallback) + ? savedConfig.fallback.indexOf(id) + : -1; + + this.unregister(id); + this.register(provider, id, priority); + + if (savedConfig) { + if (wasDefault) { + savedConfig.default = id; + } + if (fallbackIndex !== -1 && Array.isArray(savedConfig.fallback) && !savedConfig.fallback.includes(id)) { + savedConfig.fallback.splice(fallbackIndex, 0, id); + } + } + } else { + this.register(provider, id, priority); + } + } + + /** + * Unregister a provider by ID + */ + unregister(id: string): boolean { + const registered = this.providers.get(id); + if (!registered) { + return false; + } + + const channel = registered.provider.manifest.channel; + this.providers.delete(id); + + // Remove from channelProviders list + const existing = this.channelProviders.get(channel) || []; + const filtered = existing.filter(providerId => providerId !== id); + if (filtered.length > 0) { + this.channelProviders.set(channel, filtered); + } else { + this.channelProviders.delete(channel); + } + + // Clean up default / fallback if this provider was configured + const config = this.channelConfig.get(channel); + if (config) { + if (config.default === id) { + config.default = filtered[0] || ''; + } + if (Array.isArray(config.fallback)) { + config.fallback = config.fallback.filter(fId => fId !== id); + } else if (config.fallback === id) { + config.fallback = filtered[1] || undefined; + } + } + + logger.info(`Unregistered provider: ${id} from channel: ${channel}`); + return true; + } + /** * Set channel configuration (default/fallback) */ @@ -197,8 +220,23 @@ class PluginRegistryClass { getDefaultProvider(channel: string): SimpleNSProvider | undefined { const config = this.channelConfig.get(channel); if (config?.default) { - return this.get(config.default); + const defaultProvider = this.get(config.default); + if (defaultProvider) { + return defaultProvider; + } } + + // If configured default is unregistered/disabled, check explicitly configured fallbacks in order + if (config?.fallback) { + const fallbackIds = Array.isArray(config.fallback) ? config.fallback : [config.fallback]; + for (const fallbackId of fallbackIds) { + const fallbackProvider = this.get(fallbackId); + if (fallbackProvider) { + return fallbackProvider; + } + } + } + // Fall back to first registered provider for channel const providers = this.getProvidersForChannel(channel); return providers[0]; @@ -209,38 +247,58 @@ class PluginRegistryClass { */ getDefaultProviderId(channel: string): string | undefined { const config = this.channelConfig.get(channel); - if (config?.default) { + if (config?.default && this.get(config.default)) { return config.default; } + + // If configured default is unregistered/disabled, check explicitly configured fallbacks in order + if (config?.fallback) { + const fallbackIds = Array.isArray(config.fallback) ? config.fallback : [config.fallback]; + for (const fallbackId of fallbackIds) { + if (this.get(fallbackId)) { + return fallbackId; + } + } + } + // Fall back to first registered provider ID for channel const ids = this.channelProviders.get(channel); return ids?.[0]; } /** - * Get fallback provider for a channel + * Get ordered fallback provider IDs for a channel (cascading fallback array) */ - getFallbackProvider(channel: string): SimpleNSProvider | undefined { + getFallbackProviderIds(channel: string): string[] { const config = this.channelConfig.get(channel); if (config?.fallback) { - return this.get(config.fallback); + if (Array.isArray(config.fallback)) { + return config.fallback; + } + return [config.fallback]; } - // Fall back to second registered provider for channel - const providers = this.getProvidersForChannel(channel); - return providers[1]; + + // Default: return registered provider IDs for channel excluding default provider + const defaultId = this.getDefaultProviderId(channel); + const ids = this.channelProviders.get(channel) || []; + return ids.filter(id => id !== defaultId); } /** - * Get fallback provider ID for a channel + * Get ordered fallback providers for a channel */ - getFallbackProviderId(channel: string): string | undefined { - const config = this.channelConfig.get(channel); - if (config?.fallback) { - return config.fallback; - } - // Fall back to second registered provider ID for channel - const ids = this.channelProviders.get(channel); - return ids?.[1]; + getFallbackProviders(channel: string): SimpleNSProvider[] { + return this.getFallbackProviderIds(channel) + .map(id => this.get(id)) + .filter((p): p is SimpleNSProvider => p !== undefined); + } + + /** + * Get first fallback provider for a channel + */ + getFallbackProvider(channel: string): SimpleNSProvider | undefined { + const providers = this.getFallbackProviders(channel); + return providers[0]; } /** @@ -271,6 +329,20 @@ class PluginRegistryClass { return this.providers.has(id); } + /** + * Get all registered provider entries + */ + getAllRegistered(): RegisteredProvider[] { + return Array.from(this.providers.values()); + } + + /** + * Get registered provider entry by ID + */ + getRegistered(id: string): RegisteredProvider | undefined { + return this.providers.get(id); + } + /** * Check if a channel has providers */ diff --git a/src/plugins/loader/router.ts b/src/plugins/loader/router.ts index c7823ba..e0ec2d0 100644 --- a/src/plugins/loader/router.ts +++ b/src/plugins/loader/router.ts @@ -5,99 +5,215 @@ */ import { PluginRegistry } from './registry.js'; -import type { DeliveryResult, BaseNotification } from '../interfaces/provider.types.js'; +import type { DeliveryResult, BaseNotification, ProviderAttempt } from '@src/types/types.js'; import { unifiedProcessorLogger as logger } from '@src/processors/unified/unified.logger.js'; import { handleSchemaValidationFailure } from '@src/processors/shared/schema-failure-handler.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; +import { consumeToken } from '@src/processors/shared/rate-limiter.js'; /** - * Try fallback provider when primary fails with non-retryable error + * Try fallback providers when primary fails with non-retryable error. + * Cascades through fallback providers in sequence until one succeeds or all fail. */ async function tryFallback( channel: string, notification: T, primaryError: DeliveryResult['error'], - currentProviderId?: string + currentProviderId?: string, + priorAttempts: ProviderAttempt[] = [] ): Promise { - const fallbackProviderId = PluginRegistry.getFallbackProviderId(channel); - if (!fallbackProviderId) { - logger.debug(`[ProviderRouter] No fallback provider for ${channel}, returning error`); + const attempts = [...priorAttempts]; + const fallbackIds = PluginRegistry.getFallbackProviderIds(channel); + if (!fallbackIds || fallbackIds.length === 0) { + logger.debug(`[ProviderRouter] No fallback providers configured for ${channel}`); return null; } - if (currentProviderId && fallbackProviderId === currentProviderId) { - logger.debug(`[ProviderRouter] Fallback provider is same as current provider (${currentProviderId}), skipping fallback`); + const currentIndex = currentProviderId ? fallbackIds.indexOf(currentProviderId) : -1; + const candidateIds = currentIndex >= 0 ? fallbackIds.slice(currentIndex + 1) : fallbackIds.filter(id => id !== currentProviderId); + if (candidateIds.length === 0) { + logger.debug(`[ProviderRouter] No further fallback providers in cascade after ${currentProviderId}`); return null; } - const fallbackProvider = PluginRegistry.getFallbackProvider(channel); - - if (!fallbackProvider) { - logger.debug(`[ProviderRouter] No fallback provider for ${channel}, returning error`); - return null; - } + let lastError = primaryError; + + for (let i = 0; i < candidateIds.length; i++) { + const fallbackId = candidateIds[i]; + const fallbackProvider = PluginRegistry.get(fallbackId); + if (!fallbackProvider) { + logger.warn(`[ProviderRouter] Fallback provider '${fallbackId}' not found in registry, continuing cascade`); + attempts.push({ + provider: fallbackId, + status: 'failed', + error_code: 'PROVIDER_NOT_FOUND', + error_message: `Fallback provider '${fallbackId}' not found in registry`, + retryable: false, + attempted_at: new Date() + }); + continue; + } - logger.debug(`[ProviderRouter] Primary provider failed, trying fallback: ${fallbackProvider.manifest.name}`); + // Check rate limiting before attempting to send via fallback + const rateLimitResult = await consumeToken(fallbackId); + if (!rateLimitResult.allowed) { + logger.warn(`[ProviderRouter] Fallback provider '${fallbackId}' is rate limited, continuing cascade`); + lastError = { + code: 'FALLBACK_RATE_LIMITED', + message: `Fallback provider '${fallbackId}' is rate limited`, + retryable: true + }; + attempts.push({ + provider: fallbackId, + status: 'rate_limited', + error_code: 'FALLBACK_RATE_LIMITED', + error_message: `Fallback provider '${fallbackId}' is rate limited`, + retryable: true, + attempted_at: new Date() + }); + continue; + } - void AdminAlertService.sendAlert('service_health', - `ℹ️ USING FALLBACK PROVIDER\n` + - `Channel: ${channel}\n` + - `Notification ID: ${notification.notification_id}\n` + - `Primary provider error: ${primaryError?.message}\n` + - `Fallback provider: ${fallbackProvider.manifest.displayName}\n` + - `Action: Check primary provider configuration if this occurs frequently.`, - { severity: 'info', notificationId: notification.notification_id, channel }); - - const schema = fallbackProvider.getNotificationSchema(); - const validationResult = schema.safeParse(notification); - - if (!validationResult.success) { - logger.error(`[${channel}] Invalid notification schema for fallback provider of channel:${channel} and provider name ${fallbackProvider.manifest.displayName}:`, - validationResult.error.issues); - - await handleSchemaValidationFailure( - notification.notification_id, - channel, - notification, - validationResult.error, - 'fallback provider' - ); - return { - success: false, - error: { + logger.info(`[ProviderRouter] Trying fallback provider [${i + 1}/${candidateIds.length}]: ${fallbackId} (${fallbackProvider.manifest.displayName})`); + + void AdminAlertService.sendAlert('service_health', + `ℹ️ USING FALLBACK PROVIDER (${i + 1}/${candidateIds.length})\n` + + `Channel: ${channel}\n` + + `Notification ID: ${notification.notification_id}\n` + + `Previous error: ${lastError?.message}\n` + + `Fallback provider: ${fallbackProvider.manifest.displayName} (${fallbackId})\n` + + `Action: Check provider configuration if this occurs frequently.`, + { severity: 'info', notificationId: notification.notification_id, channel }); + + const schema = fallbackProvider.getNotificationSchema(); + const validationResult = schema.safeParse(notification); + + if (!validationResult.success) { + logger.error(`[${channel}] Invalid notification schema for fallback provider '${fallbackId}':`, + validationResult.error.issues); + + await handleSchemaValidationFailure( + notification.notification_id, + channel, + notification, + validationResult.error, + `fallback provider ${fallbackId}` + ); + const schemaErrMsg = validationResult.error.issues.map(iss => `${iss.path.join('.')}: ${iss.message}`).join(', '); + lastError = { code: 'FALLBACK_SCHEMA_VALIDATION_ERROR', - message: 'Fallback provider schema validation failed', + message: `Fallback provider ${fallbackId} schema validation failed: ${schemaErrMsg}`, retryable: false - } - }; - } + }; + attempts.push({ + provider: fallbackId, + status: 'failed', + error_code: 'FALLBACK_SCHEMA_VALIDATION_ERROR', + error_message: lastError.message, + retryable: false, + attempted_at: new Date() + }); + continue; + } - const fallbackResult = await fallbackProvider.send(notification); + const startTime = Date.now(); + try { + const fallbackResult = await fallbackProvider.send(notification); + const duration_ms = Date.now() - startTime; + if (fallbackResult.success) { + logger.success(`[${channel}] Notification delivered via fallback provider: ${fallbackId}`); + attempts.push({ + provider: fallbackId, + status: 'delivered', + attempted_at: new Date(), + duration_ms + }); + return { + ...fallbackResult, + provider: fallbackId, + messageId: fallbackResult.messageId || `fallback-${fallbackId}`, + attempts + }; + } - if (fallbackResult.success) { - return fallbackResult; + lastError = fallbackResult.error || { + code: 'FALLBACK_SEND_FAILED', + message: `Fallback provider ${fallbackId} send failed without error details`, + retryable: false + }; + attempts.push({ + provider: fallbackId, + status: 'failed', + error_code: lastError.code, + error_message: lastError.message, + retryable: lastError.retryable, + attempted_at: new Date(), + duration_ms + }); + } catch (sendErr) { + const duration_ms = Date.now() - startTime; + logger.error(`[${channel}] Error sending via fallback provider ${fallbackId}:`, sendErr); + lastError = { + code: 'FALLBACK_EXECUTION_ERROR', + message: sendErr instanceof Error ? sendErr.message : String(sendErr), + retryable: false + }; + attempts.push({ + provider: fallbackId, + status: 'failed', + error_code: lastError.code, + error_message: lastError.message, + retryable: false, + attempted_at: new Date(), + duration_ms + }); + } } - // Both failed - return last error + // All fallbacks failed in cascade void AdminAlertService.sendAlert('failed_notification', - `🔴 ALL PROVIDERS FAILED\n` + + `🔴 ALL PROVIDERS IN CASCADE FAILED\n` + `Channel: ${channel}\n` + `Notification ID: ${notification.notification_id}\n` + - `Primary provider error: ${primaryError?.message}\n` + - `Fallback error: ${fallbackResult.error?.message}\n` + - `Action: Verify provider credentials. Check SMTP/API connectivity. Review simplens.config.yaml.`, + `Attempted Fallbacks: ${candidateIds.join(', ')}\n` + + `Last error: ${lastError?.message}\n` + + `Action: Verify all provider credentials and connectivity for channel ${channel}.`, { severity: 'critical', notificationId: notification.notification_id, channel }); return { success: false, error: { code: 'ALL_PROVIDERS_FAILED', - message: `All providers failed. Last error: ${fallbackResult.error?.message || 'Unknown'}`, + message: `All providers in cascade failed. Last error: ${lastError?.message || 'Unknown'}`, retryable: false, }, + attempts }; } +/** + * Get ordered provider cascade for a channel starting from currentProviderId (or default provider). + */ +export function getProviderCascade( + channel: string, + currentProviderId?: string +): string[] { + const defaultId = PluginRegistry.getDefaultProviderId(channel); + const fallbackIds = PluginRegistry.getFallbackProviderIds(channel) || []; + const allProviders = defaultId ? [defaultId, ...fallbackIds] : [...fallbackIds]; + + if (!currentProviderId) { + return allProviders; + } + + const currentIndex = allProviders.indexOf(currentProviderId); + if (currentIndex >= 0) { + return allProviders.slice(currentIndex); + } + + return allProviders; +} + /** * Resolve a fallback provider ID for retry exhaustion handoff. * Returns undefined when no distinct fallback provider is available. @@ -106,13 +222,22 @@ export function resolveFallbackProviderId( channel: string, currentProviderId?: string ): string | undefined { - const fallbackProviderId = PluginRegistry.getFallbackProviderId(channel); + const fallbackIds = PluginRegistry.getFallbackProviderIds(channel); + if (!fallbackIds || fallbackIds.length === 0) return undefined; + if (!currentProviderId) return fallbackIds[0]; + + // If currentProviderId is in the cascade, advance to the next fallback provider + const currentIndex = fallbackIds.indexOf(currentProviderId); + if (currentIndex >= 0 && currentIndex + 1 < fallbackIds.length) { + return fallbackIds[currentIndex + 1]; + } - if (!fallbackProviderId || fallbackProviderId === currentProviderId) { - return undefined; + // If current was primary, return first fallback + if (currentIndex === -1) { + return fallbackIds[0]; } - return fallbackProviderId; + return undefined; } /** @@ -126,19 +251,52 @@ export async function sendWithFallback( channel: string, notification: T ): Promise { + const attempts: ProviderAttempt[] = []; + // 0. Use explicit provider if specified if (notification.provider) { logger.debug(`[ProviderRouter] Using explicit provider: ${notification.provider}`); - const result = await sendToProvider(notification.provider, notification); + const providerId = notification.provider; + const startTime = Date.now(); + const result = await sendToProvider(providerId, notification); + const duration_ms = Date.now() - startTime; + + if (result.success) { + attempts.push({ + provider: providerId, + status: 'delivered', + attempted_at: new Date(), + duration_ms + }); + return { + ...result, + provider: providerId, + attempts + }; + } - // If success or retryable error, return as-is - if (result.success || result.error?.retryable) { - return result; + attempts.push({ + provider: providerId, + status: 'failed', + error_code: result.error?.code, + error_message: result.error?.message, + retryable: result.error?.retryable, + attempted_at: new Date(), + duration_ms + }); + + // If retryable error, return as-is without fallback + if (result.error?.retryable) { + return { + ...result, + provider: providerId, + attempts + }; } // Non-retryable failure - try fallback provider - const fallbackResult = await tryFallback(channel, notification, result.error, notification.provider); - return fallbackResult ?? result; + const fallbackResult = await tryFallback(channel, notification, result.error, providerId, attempts); + return fallbackResult ?? { ...result, provider: providerId, attempts }; } const defaultProvider = PluginRegistry.getDefaultProvider(channel); @@ -151,21 +309,48 @@ export async function sendWithFallback( message: `No provider configured for channel: ${channel}`, retryable: false, }, + attempts }; } - const defaultProviderId = PluginRegistry.getDefaultProviderId(channel); + const defaultProviderId = PluginRegistry.getDefaultProviderId(channel) || 'default'; // Try default provider + const startTime = Date.now(); const result = await defaultProvider.send(notification); + const duration_ms = Date.now() - startTime; if (result.success) { - return result; + attempts.push({ + provider: defaultProviderId, + status: 'delivered', + attempted_at: new Date(), + duration_ms + }); + return { + ...result, + provider: defaultProviderId, + attempts + }; } + attempts.push({ + provider: defaultProviderId, + status: 'failed', + error_code: result.error?.code, + error_message: result.error?.message, + retryable: result.error?.retryable, + attempted_at: new Date(), + duration_ms + }); + // If error is retryable, don't fallback - let SimpleNS retry with same provider if (result.error?.retryable) { - return result; + return { + ...result, + provider: defaultProviderId, + attempts + }; } // Try fallback provider @@ -173,9 +358,10 @@ export async function sendWithFallback( channel, notification, result.error, - defaultProviderId + defaultProviderId, + attempts ); - return fallbackResult ?? result; + return fallbackResult ?? { ...result, provider: defaultProviderId, attempts }; } /** @@ -234,23 +420,13 @@ export function validateNotification( channel: string, notification: unknown ): { success: true; data: unknown } | { success: false; error: string } { - const provider = PluginRegistry.getDefaultProvider(channel); + const providerId = PluginRegistry.getDefaultProviderId(channel); - if (!provider) { + if (!providerId) { return { success: false, error: `No provider for channel: ${channel}` }; } - const schema = provider.getNotificationSchema(); - const result = schema.safeParse(notification); - - if (result.success) { - return { success: true, data: result.data }; - } - - return { - success: false, - error: result.error.issues.map(i => `${i.path.join('.')}: ${i.message}`).join(', '), - }; + return validateNotificationForProvider(providerId, notification); } /** diff --git a/src/plugins/services/channel-routing.service.ts b/src/plugins/services/channel-routing.service.ts new file mode 100644 index 0000000..6873f9f --- /dev/null +++ b/src/plugins/services/channel-routing.service.ts @@ -0,0 +1,165 @@ +/** + * Channel Routing Service + * + * Manages channel default/cascading fallback provider configuration, + * dynamic Kafka partition scaling, and Redis pub/sub synchronization. + */ + +import ChannelRouting from '@src/database/models/channel-routing.models.js'; +import Provider from '@src/database/models/provider.models.js'; +import { PluginRegistry } from '@src/plugins/loader/registry.js'; +import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; +import { expandTopicPartitions, ensureChannelTopic } from '@src/config/kafka.config.js'; +import type { channel_routing_document } from '@src/types/types.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +export class ChannelRoutingService { + /** + * List all configured channel routings + */ + public static async listChannelRoutings(): Promise { + return (await ChannelRouting.find().sort({ channel: 1 }).lean()) as channel_routing_document[]; + } + + /** + * Get routing configuration for a specific channel + */ + public static async getChannelRouting(channel: string): Promise { + return (await ChannelRouting.findOne({ channel }).lean()) as channel_routing_document | null; + } + + /** + * Set or update channel routing configuration and expand Kafka partitions if requested + */ + public static async setChannelRouting( + channel: string, + data: { + default_provider_id: string; + fallback_provider_ids?: string[]; + partitions?: number; + } + ): Promise { + logger.info(`Updating channel routing for '${channel}'...`); + + // 1. Validate default provider + const defaultProvider = await Provider.findOne({ id: data.default_provider_id }); + if (!defaultProvider) { + throw new Error(`Default provider '${data.default_provider_id}' not found.`); + } + if (defaultProvider.channel !== channel) { + throw new Error( + `Default provider '${data.default_provider_id}' is configured for channel '${defaultProvider.channel}', not '${channel}'.` + ); + } + + // 2. Validate fallback providers + const fallbacks = data.fallback_provider_ids || []; + for (const fId of fallbacks) { + const fProvider = await Provider.findOne({ id: fId }); + if (!fProvider) { + throw new Error(`Fallback provider '${fId}' not found.`); + } + if (fProvider.channel !== channel) { + throw new Error( + `Fallback provider '${fId}' is configured for channel '${fProvider.channel}', not '${channel}'.` + ); + } + } + + // 3. Handle dynamic Kafka topic creation and partition scaling + const existing = await ChannelRouting.findOne({ channel }); + const currentPartitions = existing?.partitions || 6; + const desiredPartitions = data.partitions !== undefined ? data.partitions : currentPartitions; + + if (existing && desiredPartitions < currentPartitions) { + throw new Error( + `Cannot decrease partitions for channel '${channel}' from ${currentPartitions} to ${desiredPartitions}. Kafka partitions can only be increased.` + ); + } + + try { + if (desiredPartitions > currentPartitions) { + logger.info( + `Requesting Kafka partition expansion for channel '${channel}' from ${currentPartitions} to ${desiredPartitions}...` + ); + await expandTopicPartitions(channel, desiredPartitions); + } else { + await ensureChannelTopic(channel, desiredPartitions); + } + } catch (kafkaErr) { + logger.error(`Failed to configure Kafka topic for channel '${channel}':`, kafkaErr); + throw new Error(`Failed to expand Kafka partitions: ${(kafkaErr as Error).message}`, { + cause: kafkaErr, + }); + } + + // 4. Save to MongoDB + const routing = await ChannelRouting.findOneAndUpdate( + { channel }, + { + channel, + default_provider_id: data.default_provider_id, + fallback_provider_ids: fallbacks, + partitions: desiredPartitions, + }, + { upsert: true, new: true } + ); + + // 5. Update local PluginRegistry + PluginRegistry.setChannelConfig(channel, { + default: data.default_provider_id, + fallback: fallbacks, + }); + + // 6. Broadcast sync event + await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { + channel, + }); + + logger.success(`Channel routing for '${channel}' updated successfully.`); + return routing.toObject() as channel_routing_document; + } + + /** + * Delete routing configuration for a channel + */ + public static async deleteChannelRouting(channel: string): Promise { + logger.info(`Deleting channel routing for '${channel}'...`); + await ChannelRouting.deleteOne({ channel }); + + PluginRegistry.setChannelConfig(channel, { + default: '', + fallback: [], + }); + + await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { + channel, + }); + logger.success(`Channel routing for '${channel}' deleted.`); + } + + /** + * Register remote synchronization handlers with Redis Pub/Sub + */ + public static registerSyncHandlers(): void { + PluginSyncService.on('CHANNEL_ROUTING_UPDATED', async (msg) => { + const { channel } = msg.payload; + if (!channel) return; + + logger.info(`Sync: Handling remote CHANNEL_ROUTING_UPDATED for '${channel}'...`); + const routing = await ChannelRouting.findOne({ channel }); + + if (routing) { + PluginRegistry.setChannelConfig(channel, { + default: routing.default_provider_id, + fallback: routing.fallback_provider_ids || [], + }); + } else { + PluginRegistry.setChannelConfig(channel, { + default: '', + fallback: [], + }); + } + }); + } +} diff --git a/src/plugins/services/npm-auth.service.ts b/src/plugins/services/npm-auth.service.ts new file mode 100644 index 0000000..d5d8b49 --- /dev/null +++ b/src/plugins/services/npm-auth.service.ts @@ -0,0 +1,433 @@ +/** + * NPM Authentication & Registry Configuration Service + * + * Manages multiple npm access tokens and enterprise registry configurations, + * encrypted storage in MongoDB, and synchronization of .plugins/.npmrc. + */ + +import { writeFileSync } from 'fs'; +import { join } from 'path'; +import system_config_model from '@src/database/models/system-config.models.js'; +import { getOrCreateEncryptionKey } from '@src/admin-alerts/key-manager.js'; +import { encrypt, decrypt } from '@src/utils/encryption.utils.js'; +import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; +import { PLUGINS_DIR, initPluginsDir } from '@src/plugins/loader/plugin-fs.js'; +import type { + npm_auth_status, + npm_registry_status, + StoredNpmAuthConfig, + StoredRegistryEntry, + DecryptedRegistryEntry +} from '@src/types/types.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +export const NPM_AUTH_CONFIG_KEY = 'npm_auth_config'; +export const NPMRC_PATH = join(PLUGINS_DIR, '.npmrc'); + +/** + * Extract npm scope from a package name if present + * E.g. "@simplens/resend" -> "@simplens", "lodash" -> undefined + */ +export function extractScope(packageName: string): string | undefined { + const match = packageName.trim().match(/^(@[a-zA-Z0-9~_.-]+)\//); + return match ? match[1].toLowerCase() : undefined; +} + +/** + * Normalize npm registry URL to ensure valid URL with trailing slash + */ +export function normalizeRegistryUrl(registryUrl?: string): string { + if (!registryUrl || !registryUrl.trim()) { + return 'https://registry.npmjs.org/'; + } + let trimmed = registryUrl.trim(); + if (!trimmed.endsWith('/')) { + trimmed += '/'; + } + return trimmed; +} + +/** + * Generate a consistent identifier for a registry entry + */ +export function generateRegistryId(scope?: string, registryUrl?: string): string { + if (scope) { + return `scope:${scope.trim().toLowerCase()}`; + } + const normalized = normalizeRegistryUrl(registryUrl); + if (!normalized.includes('registry.npmjs.org')) { + try { + return `url:${new URL(normalized).host.toLowerCase()}`; + } catch { + // fallback + } + } + return 'default'; +} + +/** + * Mask an authentication token for safe public/UI presentation + */ +export function maskToken(token: string): string { + const t = token.trim(); + return t.length > 8 ? `${t.slice(0, 4)}****${t.slice(-4)}` : '****'; +} + +/** + * Format a registry URL into an .npmrc auth token line + * E.g. "https://registry.npmjs.org/" -> "//registry.npmjs.org/:_authToken=..." + */ +export function toNpmrcTokenLine(registryUrl: string, token: string): string { + try { + const url = new URL(registryUrl); + let hostPath = url.host + url.pathname; + if (!hostPath.endsWith('/')) { + hostPath += '/'; + } + return `//${hostPath}:_authToken=${token.trim()}`; + } catch { + return `//registry.npmjs.org/:_authToken=${token.trim()}`; + } +} + +export class NpmAuthService { + /** + * Retrieve all stored registry entries, migrating legacy single-token documents if present + */ + private static async getRawStoredEntries(): Promise { + try { + const doc = await system_config_model.findOne({ key: NPM_AUTH_CONFIG_KEY }).lean(); + if (!doc?.value) { + return []; + } + + const stored = doc.value as StoredNpmAuthConfig; + const entries: StoredRegistryEntry[] = []; + + if (Array.isArray(stored.registries) && stored.registries.length > 0) { + entries.push(...stored.registries); + } else if (stored.encrypted_token) { + // Migrate legacy single-token document + const regUrl = normalizeRegistryUrl(stored.registry_url); + entries.push({ + id: generateRegistryId(undefined, regUrl), + registry_url: regUrl, + encrypted_token: stored.encrypted_token, + updated_at: new Date().toISOString(), + }); + } + + return entries; + } catch (err) { + logger.error('Failed to read npm registries from MongoDB:', err); + return []; + } + } + + /** + * Retrieve all decrypted registry configurations from MongoDB + environment variables + */ + public static async getDecryptedRegistries(): Promise { + const entries = await this.getRawStoredEntries(); + const result: DecryptedRegistryEntry[] = []; + + if (entries.length > 0) { + try { + const key = await getOrCreateEncryptionKey(); + for (const entry of entries) { + try { + const token = decrypt(entry.encrypted_token, key); + result.push({ + id: entry.id, + scope: entry.scope, + registry_url: normalizeRegistryUrl(entry.registry_url), + token, + updated_at: entry.updated_at, + }); + } catch (decErr) { + logger.error(`Failed to decrypt token for registry ${entry.id}:`, decErr); + } + } + } catch (keyErr) { + logger.error('Failed to get encryption key for npm registries:', keyErr); + } + } + + // Include environment variable fallback if not already present + const envToken = process.env.NPM_TOKEN || process.env.NPM_AUTH_TOKEN; + if (envToken) { + const envRegUrl = normalizeRegistryUrl(process.env.NPM_REGISTRY_URL); + const exists = result.some(r => r.registry_url === envRegUrl && !r.scope); + if (!exists) { + result.push({ + id: 'env-default', + registry_url: envRegUrl, + token: envToken, + updated_at: new Date().toISOString(), + }); + } + } + + return result; + } + + /** + * Retrieve active npm auth credentials (backwards-compatible helper) + */ + public static async getDecryptedToken(): Promise<{ token: string; registry_url: string } | null> { + const registries = await this.getDecryptedRegistries(); + if (registries.length > 0) { + const primary = registries[0]; + return { + token: primary.token, + registry_url: primary.registry_url, + }; + } + return null; + } + + /** + * Find matching credentials for a given package name and/or target registry URL + */ + public static async findMatchingAuth( + packageName: string, + registryUrl?: string + ): Promise { + const registries = await this.getDecryptedRegistries(); + if (registries.length === 0) { + return null; + } + + const scope = extractScope(packageName); + const targetUrl = registryUrl ? normalizeRegistryUrl(registryUrl) : undefined; + + // 1. Try matching by package scope (e.g. "@myenterprise") + if (scope) { + const scopeMatch = registries.find(r => r.scope?.toLowerCase() === scope); + if (scopeMatch) { + return scopeMatch; + } + } + + // 2. Try matching by target registry URL host + if (targetUrl) { + try { + const targetHost = new URL(targetUrl).host.toLowerCase(); + const urlMatch = registries.find(r => { + try { + return new URL(r.registry_url).host.toLowerCase() === targetHost; + } catch { + return false; + } + }); + if (urlMatch) { + return urlMatch; + } + } catch { + // invalid URL ignore + } + } + + // 3. Fallback: If target is default npmjs or unspecifed, find default/global registry + const defaultMatch = registries.find(r => !r.scope && r.registry_url.includes('registry.npmjs.org')); + if (defaultMatch) { + return defaultMatch; + } + + // 4. Return first registry if no specific match and it's a global entry + const anyGlobal = registries.find(r => !r.scope); + return anyGlobal || null; + } + + /** + * Get public status of all configured registries (with masked tokens) + */ + public static async getAuthStatus(): Promise { + const registries = await this.getDecryptedRegistries(); + if (registries.length === 0) { + return { + is_configured: false, + registries: [], + }; + } + + const registryStatuses: npm_registry_status[] = registries.map(r => ({ + id: r.id, + scope: r.scope, + registry_url: r.registry_url, + masked_token: maskToken(r.token), + updated_at: r.updated_at, + })); + + const primary = registries[0]; + return { + is_configured: true, + registries: registryStatuses, + default_registry: primary.registry_url, + masked_token: maskToken(primary.token), + }; + } + + /** + * Save and encrypt npm auth configuration for a scope or registry in MongoDB + */ + public static async saveNpmAuth( + token: string, + registryUrl?: string, + scope?: string + ): Promise { + const cleanToken = token.trim(); + const cleanRegistry = normalizeRegistryUrl(registryUrl); + const cleanScope = scope?.trim() ? (scope.startsWith('@') ? scope.trim().toLowerCase() : `@${scope.trim().toLowerCase()}`) : undefined; + const entryId = generateRegistryId(cleanScope, cleanRegistry); + + const key = await getOrCreateEncryptionKey(); + const encryptedToken = encrypt(cleanToken, key); + + const currentEntries = await this.getRawStoredEntries(); + const existingIndex = currentEntries.findIndex(e => e.id === entryId || (cleanScope && e.scope === cleanScope)); + + const newEntry: StoredRegistryEntry = { + id: entryId, + scope: cleanScope, + registry_url: cleanRegistry, + encrypted_token: encryptedToken, + updated_at: new Date().toISOString(), + }; + + if (existingIndex >= 0) { + currentEntries[existingIndex] = newEntry; + } else { + currentEntries.push(newEntry); + } + + const storedValue: StoredNpmAuthConfig = { + registries: currentEntries, + // Clear legacy single fields to prevent ambiguity + encrypted_token: undefined, + registry_url: undefined, + }; + + await system_config_model.findOneAndUpdate( + { key: NPM_AUTH_CONFIG_KEY }, + { key: NPM_AUTH_CONFIG_KEY, value: storedValue }, + { upsert: true, new: true } + ); + + await this.syncNpmrc(); + await PluginSyncService.publish('NPM_AUTH_UPDATED', {}); + + logger.success(`npm auth configuration saved and broadcasted for ${cleanScope || cleanRegistry} (${entryId})`); + return this.getAuthStatus(); + } + + /** + * Delete saved npm auth configuration by ID, scope, or delete all + */ + public static async deleteNpmAuth(idOrScope?: string): Promise { + if (!idOrScope || idOrScope === 'all') { + await system_config_model.deleteOne({ key: NPM_AUTH_CONFIG_KEY }); + logger.info('All npm auth configurations removed'); + } else { + const currentEntries = await this.getRawStoredEntries(); + const target = idOrScope.trim().toLowerCase(); + const filtered = currentEntries.filter( + e => e.id.toLowerCase() !== target && e.scope?.toLowerCase() !== target && e.registry_url.toLowerCase() !== target + ); + + const storedValue: StoredNpmAuthConfig = { + registries: filtered, + encrypted_token: undefined, + registry_url: undefined, + }; + + await system_config_model.findOneAndUpdate( + { key: NPM_AUTH_CONFIG_KEY }, + { key: NPM_AUTH_CONFIG_KEY, value: storedValue }, + { upsert: true, new: true } + ); + logger.info(`Removed npm auth configuration for ${idOrScope}`); + } + + await this.syncNpmrc(); + await PluginSyncService.publish('NPM_AUTH_UPDATED', {}); + } + + /** + * Generate and write .plugins/.npmrc containing all scoped and host credentials + * Supports optional inline scope/registry/token for one-off private installs + */ + public static async syncNpmrc(temporaryOverride?: { + scope?: string; + registryUrl?: string; + token?: string; + }): Promise { + initPluginsDir(); + + const lines: string[] = [ + '# Auto-generated by SimpleNS - do not edit manually', + 'package-lock=false', + ]; + + const registries = await this.getDecryptedRegistries(); + const writtenHosts = new Set(); + const writtenScopes = new Set(); + + // 1. Process all persistent registries + for (const reg of registries) { + if (reg.scope && !writtenScopes.has(reg.scope)) { + if (!reg.registry_url.includes('registry.npmjs.org')) { + lines.push(`${reg.scope}:registry=${reg.registry_url}`); + } + writtenScopes.has(reg.scope); + } + + const hostLine = toNpmrcTokenLine(reg.registry_url, reg.token); + if (!writtenHosts.has(hostLine)) { + lines.push(hostLine); + writtenHosts.add(hostLine); + } + + // If global custom registry with no scope + if (!reg.scope && !reg.registry_url.includes('registry.npmjs.org')) { + lines.push(`registry=${reg.registry_url}`); + } + } + + // 2. Temporary/Inline override (e.g. for a specific scoped install) + if (temporaryOverride?.token) { + const targetRegistry = normalizeRegistryUrl(temporaryOverride.registryUrl); + if (temporaryOverride.scope) { + const cleanScope = temporaryOverride.scope.startsWith('@') + ? temporaryOverride.scope.toLowerCase() + : `@${temporaryOverride.scope.toLowerCase()}`; + lines.push(`${cleanScope}:registry=${targetRegistry}`); + } else if (!targetRegistry.includes('registry.npmjs.org')) { + lines.push(`registry=${targetRegistry}`); + } + const overrideHostLine = toNpmrcTokenLine(targetRegistry, temporaryOverride.token); + if (!writtenHosts.has(overrideHostLine)) { + lines.push(overrideHostLine); + writtenHosts.add(overrideHostLine); + } + } + + const content = lines.join('\n') + '\n'; + try { + writeFileSync(NPMRC_PATH, content, { encoding: 'utf-8', mode: 0o600 }); + logger.debug(`Synchronized .npmrc in ${PLUGINS_DIR} with ${registries.length} registries`); + } catch (err) { + logger.error('Failed to write .npmrc:', err); + } + } + + /** + * Register remote synchronization handlers with Redis Pub/Sub + */ + public static registerSyncHandlers(): void { + PluginSyncService.on('NPM_AUTH_UPDATED', async () => { + logger.info('Sync: Received NPM_AUTH_UPDATED event, refreshing .npmrc...'); + await NpmAuthService.syncNpmrc(); + }); + } +} diff --git a/src/plugins/services/plugin-manager.service.ts b/src/plugins/services/plugin-manager.service.ts new file mode 100644 index 0000000..ada8e4e --- /dev/null +++ b/src/plugins/services/plugin-manager.service.ts @@ -0,0 +1,235 @@ +/** + * Plugin Manager Service + * + * Handles npm package installation, version changes, uninstallation, + * metadata extraction, database records, and Redis pub/sub synchronization. + */ + +import Plugin from '@src/database/models/plugin.models.js'; +import Provider from '@src/database/models/provider.models.js'; +import ChannelRouting from '@src/database/models/channel-routing.models.js'; +import { PluginRegistry } from '@src/plugins/loader/registry.js'; +import { + installNpmPackage, + uninstallNpmPackage, + extractPackageManifest, +} from '@src/plugins/loader/plugin-fs.js'; +import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; +import { ProviderManagerService } from '@src/plugins/services/provider-manager.service.js'; +import type { plugin_document, ProviderManifest, install_plugin_payload } from '@src/types/types.js'; +import { NpmAuthService, extractScope } from './npm-auth.service.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; + +export class PluginManagerService { + /** + * Helper to install package, extract manifest, upsert/update Plugin record, and broadcast sync event + */ + private static async saveAndBroadcastPlugin( + packageName: string, + version: string | undefined, + syncAction: 'PLUGIN_INSTALLED' | 'PLUGIN_VERSION_CHANGED', + upsert = false + ): Promise { + return await PluginSyncService.runWithLock(async () => { + try { + installNpmPackage(packageName, version); + + let manifest: ProviderManifest; + let authoritativeVersion: string; + try { + const extracted = await extractPackageManifest(packageName); + manifest = extracted.manifest; + authoritativeVersion = extracted.version; + } catch (validationErr) { + logger.warn(`Plugin validation failed for '${packageName}'. Rolling back by uninstalling...`); + try { + uninstallNpmPackage(packageName); + } catch (uninstallErr) { + logger.error(`Rollback failed for '${packageName}':`, uninstallErr); + } + throw validationErr; + } + + const plugin = await Plugin.findOneAndUpdate( + { name: packageName }, + { + name: packageName, + version: authoritativeVersion, + status: 'installed', + manifest, + error: undefined, + }, + upsert ? { upsert: true, new: true } : { new: true } + ); + + await PluginSyncService.publish(syncAction, { + plugin_name: packageName, + version: authoritativeVersion, + }); + + logger.success( + `Plugin '${packageName}' ${upsert ? '@' + authoritativeVersion + ' installed' : 'updated to version ' + authoritativeVersion} successfully.` + ); + return plugin!.toObject() as plugin_document; + } catch (err) { + logger.error(`Failed to process plugin '${packageName}':`, err); + throw err; + } + }); + } + + /** + * Install an npm package, extract its manifest, persist to MongoDB, and broadcast. + */ + public static async installPlugin( + packageName: string, + version?: string, + auth?: install_plugin_payload['auth'] + ): Promise { + const scope = extractScope(packageName); + if (auth?.token) { + const targetScope = auth.scope || scope; + if (auth.save_token !== false) { + await NpmAuthService.saveNpmAuth(auth.token, auth.registry_url, targetScope); + } else { + await NpmAuthService.syncNpmrc({ + scope: targetScope, + registryUrl: auth.registry_url, + token: auth.token, + }); + } + } + + logger.info(`Installing plugin package '${packageName}' (version: ${version || 'latest'})...`); + return this.saveAndBroadcastPlugin(packageName, version, 'PLUGIN_INSTALLED', true); + } + + /** + * Change/upgrade/downgrade the version of an already installed plugin. + */ + public static async changePluginVersion( + packageName: string, + version: string + ): Promise { + logger.info(`Changing version of plugin '${packageName}' to '${version}'...`); + + const existing = await Plugin.findOne({ name: packageName }); + if (!existing) { + throw new Error(`Plugin '${packageName}' is not installed.`); + } + + return this.saveAndBroadcastPlugin(packageName, version, 'PLUGIN_VERSION_CHANGED', false); + } + + /** + * Uninstall a plugin and remove its dependent routing and provider records. + */ + public static async uninstallPlugin(packageName: string): Promise { + logger.info(`Uninstalling plugin '${packageName}'...`); + + const existing = await Plugin.findOne({ name: packageName }); + if (!existing) { + throw new Error(`Plugin '${packageName}' not found.`); + } + + await PluginSyncService.runWithLock(async () => { + try { + uninstallNpmPackage(packageName); + + const dependentProviders = await Provider.find({ plugin_name: packageName }) + .select({ id: 1 }) + .lean(); + const providerIds = dependentProviders.map((provider) => provider.id); + + if (providerIds.length > 0) { + const dependentRoutings = await ChannelRouting.find({ + $or: [ + { default_provider_id: { $in: providerIds } }, + { fallback_provider_ids: { $in: providerIds } }, + ], + }).select({ channel: 1 }).lean(); + const routingChannels = dependentRoutings.map((routing) => routing.channel); + + if (routingChannels.length > 0) { + await ChannelRouting.deleteMany({ channel: { $in: routingChannels } }); + for (const channel of routingChannels) { + PluginRegistry.setChannelConfig(channel, { default: '', fallback: [] }); + await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { channel }); + } + } + + await Provider.deleteMany({ plugin_name: packageName }); + ProviderManagerService.clearCachedCredentials(providerIds); + for (const providerId of providerIds) { + PluginRegistry.unregister(providerId); + await PluginSyncService.publish('PROVIDER_DELETED', { provider_id: providerId }); + } + } + + await Plugin.deleteOne({ name: packageName }); + + await PluginSyncService.publish('PLUGIN_UNINSTALLED', { + plugin_name: packageName, + }); + + logger.success(`Plugin '${packageName}' uninstalled successfully.`); + } catch (err) { + logger.error(`Failed to uninstall plugin '${packageName}':`, err); + throw err; + } + }); + } + + /** + * List all installed plugins. + */ + public static async listInstalledPlugins(): Promise { + return (await Plugin.find().sort({ name: 1 }).lean()) as plugin_document[]; + } + + /** + * Get a single installed plugin by name. + */ + public static async getPlugin(name: string): Promise { + return (await Plugin.findOne({ name }).lean()) as plugin_document | null; + } + + /** + * Register remote synchronization handlers with Redis Pub/Sub + */ + public static registerSyncHandlers(): void { + const handleRemoteInstallOrUpdate = async (actionDesc: string, payload: { plugin_name?: string; version?: string }) => { + const { plugin_name, version } = payload; + if (!plugin_name) return; + logger.info(`Sync: ${actionDesc} for '${plugin_name}' (${version || 'latest'})...`); + await PluginSyncService.runWithLock(async () => { + try { + installNpmPackage(plugin_name, version); + } catch (err) { + logger.error(`Failed to sync remote ${actionDesc} for '${plugin_name}':`, err); + } + }); + }; + + PluginSyncService.on('PLUGIN_INSTALLED', async (msg) => { + await handleRemoteInstallOrUpdate('Installing remote plugin', msg.payload); + }); + + PluginSyncService.on('PLUGIN_VERSION_CHANGED', async (msg) => { + await handleRemoteInstallOrUpdate('Changing remote plugin version', msg.payload); + }); + + PluginSyncService.on('PLUGIN_UNINSTALLED', async (msg) => { + const { plugin_name } = msg.payload; + if (!plugin_name) return; + logger.info(`Sync: Uninstalling remote plugin '${plugin_name}'...`); + await PluginSyncService.runWithLock(async () => { + try { + uninstallNpmPackage(plugin_name); + } catch (err) { + logger.error(`Failed to sync remote uninstall of '${plugin_name}':`, err); + } + }); + }); + } +} diff --git a/src/plugins/services/provider-manager.service.ts b/src/plugins/services/provider-manager.service.ts new file mode 100644 index 0000000..18ee4f8 --- /dev/null +++ b/src/plugins/services/provider-manager.service.ts @@ -0,0 +1,531 @@ +/** + * Provider Manager Service + * + * Handles CRUD operations for provider instances, envelope encryption of credentials, + * live connection testing, registry loading, and Redis synchronization. + */ + +import Provider from '@src/database/models/provider.models.js'; +import Plugin from '@src/database/models/plugin.models.js'; +import ChannelRouting from '@src/database/models/channel-routing.models.js'; +import { ChannelRoutingService } from '@src/plugins/services/channel-routing.service.js'; +import { + encryptCredentials, + decryptCredentials, +} from '@src/plugins/crypto/keypair-manager.js'; +import { importAndInstantiateProvider } from '@src/plugins/loader/plugin-fs.js'; +import { PluginRegistry } from '@src/plugins/loader/registry.js'; +import { PluginSyncService } from '@src/plugins/sync/plugin-sync.service.js'; +import type { + provider_document, + ProviderConfig, + ProviderOptions, + ProviderResponseDto, + SimpleNSProvider, + encrypted_credentials, +} from '@src/types/types.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; +import { resetRateLimiter } from '@src/processors/shared/rate-limiter.js'; + +export type { ProviderResponseDto } from '@src/types/types.js'; + +export class ProviderManagerService { + private static readonly credentialCacheTtlMs = 5 * 60 * 1000; + private static readonly credentialCache = new Map< + string, + { credentials: Record; expiresAt: number } + >(); + + private static getCachedCredentials(id: string): Record | undefined { + const cached = ProviderManagerService.credentialCache.get(id); + if (!cached || cached.expiresAt <= Date.now()) { + ProviderManagerService.credentialCache.delete(id); + return undefined; + } + return cached.credentials; + } + + private static cacheCredentials(id: string, credentials: Record): void { + ProviderManagerService.credentialCache.set(id, { + credentials, + expiresAt: Date.now() + ProviderManagerService.credentialCacheTtlMs, + }); + } + + private static invalidateCredentialCache(id: string): void { + ProviderManagerService.credentialCache.delete(id); + } + + public static clearCachedCredentials(providerIds: string[]): void { + for (const providerId of providerIds) { + ProviderManagerService.invalidateCredentialCache(providerId); + } + } + + /** + * Create a new provider instance with envelope-encrypted credentials + */ + public static async createProvider(data: { + id: string; + plugin_name: string; + credentials: Record; + options?: ProviderOptions; + enabled?: boolean; + }): Promise { + logger.info(`Creating provider '${data.id}' using plugin '${data.plugin_name}'...`); + + const existing = await Provider.findOne({ id: data.id }); + if (existing) { + throw new Error(`Provider with ID '${data.id}' already exists.`); + } + + const plugin = await Plugin.findOne({ name: data.plugin_name }); + if (!plugin) { + throw new Error(`Plugin '${data.plugin_name}' is not installed.`); + } + + const encryptedCredentials = await encryptCredentials(data.credentials); + + const providerDoc = await Provider.create({ + id: data.id, + plugin_name: data.plugin_name, + channel: plugin.manifest.channel, + enabled: data.enabled !== false, + credentials: encryptedCredentials, + options: data.options || {}, + }); + + // If enabled, load into current process registry + if (providerDoc.enabled) { + try { + await ProviderManagerService.loadAndRegisterProvider(providerDoc.toObject() as provider_document); + } catch (err) { + logger.warn(`Provider '${data.id}' registered in DB, but failed local initialization:`, { + error: err instanceof Error ? err.message : String(err), + }); + } + + // Automatically create ChannelRouting if none exists for this channel yet + try { + const existingRouting = await ChannelRouting.findOne({ channel: providerDoc.channel }); + if (!existingRouting) { + logger.info(`No channel routing found for '${providerDoc.channel}'. Automatically creating default routing with provider '${data.id}'...`); + await ChannelRoutingService.setChannelRouting(providerDoc.channel, { + default_provider_id: data.id, + fallback_provider_ids: [], + partitions: 6, + }); + } + } catch (routingErr) { + logger.warn(`Failed to auto-create channel routing for '${providerDoc.channel}':`, { + error: routingErr instanceof Error ? routingErr.message : String(routingErr), + }); + } + } + + await PluginSyncService.publish('PROVIDER_UPSERTED', { + provider_id: data.id, + }); + + logger.success(`Provider '${data.id}' created successfully.`); + + return { + _id: providerDoc._id.toString(), + id: providerDoc.id, + plugin_name: providerDoc.plugin_name, + channel: providerDoc.channel, + enabled: providerDoc.enabled, + options: providerDoc.options as ProviderOptions | undefined, + credentials_configured: true, + created_at: providerDoc.created_at, + updated_at: providerDoc.updated_at, + }; + } + + /** + * Update an existing provider instance + */ + public static async updateProvider( + id: string, + updates: { + credentials?: Record; + options?: ProviderOptions; + enabled?: boolean; + } + ): Promise { + logger.info(`Updating provider '${id}'...`); + + const providerDoc = await Provider.findOne({ id }); + if (!providerDoc) { + throw new Error(`Provider '${id}' not found.`); + } + + if (updates.enabled !== undefined) { + providerDoc.enabled = updates.enabled; + } + if (updates.options !== undefined) { + providerDoc.options = updates.options; + } + if (updates.credentials && Object.keys(updates.credentials).length > 0) { + providerDoc.credentials = await encryptCredentials(updates.credentials); + ProviderManagerService.invalidateCredentialCache(id); + } + + await providerDoc.save(); + + if (providerDoc.enabled) { + try { + await ProviderManagerService.loadAndRegisterProvider(providerDoc.toObject() as provider_document); + } catch (err) { + logger.warn(`Provider '${id}' updated in DB, but failed local reload:`, { + error: err instanceof Error ? err.message : String(err), + }); + } + } else { + PluginRegistry.unregister(id); + + // If disabling a provider that was currently default in ChannelRouting, promote an active fallback + try { + const routings = await ChannelRouting.find({ default_provider_id: id }).lean(); + for (const routing of routings) { + const enabledProviders = await Provider.find({ + channel: routing.channel, + enabled: true, + id: { $ne: id }, + }).lean(); + const enabledProviderIds = new Set(enabledProviders.map((p) => p.id)); + + const remainingFallbacks = (routing.fallback_provider_ids || []).filter( + (pid: string) => pid !== id + ); + const nextDefault = remainingFallbacks.find((pid: string) => enabledProviderIds.has(pid)) + || enabledProviders[0]?.id; + + if (nextDefault) { + const newFallbacks = remainingFallbacks.filter((pid: string) => pid !== nextDefault); + await ChannelRouting.updateOne( + { _id: routing._id }, + { + $set: { + default_provider_id: nextDefault, + fallback_provider_ids: newFallbacks, + }, + } + ); + PluginRegistry.setChannelConfig(routing.channel, { + default: nextDefault, + fallback: newFallbacks, + }); + await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { channel: routing.channel }); + logger.info(`Promoted fallback provider '${nextDefault}' as default for channel '${routing.channel}' upon disabling '${id}'.`); + } + } + } catch (routingErr) { + logger.warn(`Failed to promote fallback routing when disabling provider '${id}':`, { + error: routingErr instanceof Error ? routingErr.message : String(routingErr), + }); + } + } + + await PluginSyncService.publish('PROVIDER_UPSERTED', { + provider_id: id, + }); + + logger.success(`Provider '${id}' updated successfully.`); + + return { + _id: providerDoc._id.toString(), + id: providerDoc.id, + plugin_name: providerDoc.plugin_name, + channel: providerDoc.channel, + enabled: providerDoc.enabled, + options: providerDoc.options as Record | undefined, + credentials_configured: true, + created_at: providerDoc.created_at, + updated_at: providerDoc.updated_at, + }; + } + + /** + * Delete a provider instance and remove it from channel routing. + */ + public static async deleteProvider(id: string): Promise { + logger.info(`Deleting provider '${id}'...`); + + const providerDoc = await Provider.findOne({ id }); + if (!providerDoc) { + throw new Error(`Provider '${id}' not found.`); + } + + const routings = await ChannelRouting.find({ + $or: [{ default_provider_id: id }, { fallback_provider_ids: id }], + }).lean(); + + for (const routing of routings) { + const remainingFallbacks = (routing.fallback_provider_ids || []).filter( + (providerId: string) => providerId !== id + ); + + let updatedDefault = routing.default_provider_id; + let updatedFallbacks = remainingFallbacks; + + if (routing.default_provider_id === id) { + const enabledProviders = await Provider.find({ + channel: routing.channel, + enabled: true, + id: { $ne: id }, + }).lean(); + const enabledProviderIds = new Set(enabledProviders.map((p) => p.id)); + + const nextDefault = remainingFallbacks.find((pid: string) => enabledProviderIds.has(pid)) + || remainingFallbacks[0] + || enabledProviders[0]?.id; + + if (nextDefault) { + const fallbacks = remainingFallbacks.filter((pid: string) => pid !== nextDefault); + updatedDefault = nextDefault; + updatedFallbacks = fallbacks; + await ChannelRouting.updateOne( + { channel: routing.channel }, + { + default_provider_id: nextDefault, + fallback_provider_ids: fallbacks, + } + ); + } else { + updatedDefault = ''; + updatedFallbacks = []; + await ChannelRouting.deleteOne({ channel: routing.channel }); + } + } else { + await ChannelRouting.updateOne( + { channel: routing.channel }, + { fallback_provider_ids: remainingFallbacks } + ); + } + + PluginRegistry.setChannelConfig(routing.channel, { + default: updatedDefault, + fallback: updatedFallbacks, + }); + await PluginSyncService.publish('CHANNEL_ROUTING_UPDATED', { + channel: routing.channel, + }); + } + + await Provider.deleteOne({ id }); + ProviderManagerService.invalidateCredentialCache(id); + PluginRegistry.unregister(id); + await resetRateLimiter(id).catch((err) => { + logger.warn(`Failed to clean up rate limiter keys for deleted provider '${id}':`, err); + }); + + await PluginSyncService.publish('PROVIDER_DELETED', { + provider_id: id, + }); + + logger.success(`Provider '${id}' deleted successfully.`); + } + + /** + * List all providers without exposing ciphertext credentials + */ + public static async listProviders(): Promise { + const providers = await Provider.find().sort({ channel: 1, 'options.priority': -1 }).lean(); + + return providers.map((p) => ({ + _id: p._id.toString(), + id: p.id, + plugin_name: p.plugin_name, + channel: p.channel, + enabled: p.enabled, + options: p.options as ProviderOptions | undefined, + credentials_configured: !!p.credentials?.encrypted_data, + created_at: p.created_at, + updated_at: p.updated_at, + })); + } + + /** + * Get single provider by ID + */ + public static async getProvider( + id: string, + includeDecrypted = false + ): Promise<(ProviderResponseDto & { decrypted_credentials?: Record }) | null> { + const providerDoc = await Provider.findOne({ id }).lean(); + if (!providerDoc) return null; + + let decrypted: Record | undefined; + if (includeDecrypted && providerDoc.credentials) { + decrypted = await decryptCredentials(providerDoc.credentials); + } + + return { + _id: providerDoc._id.toString(), + id: providerDoc.id, + plugin_name: providerDoc.plugin_name, + channel: providerDoc.channel, + enabled: providerDoc.enabled, + options: providerDoc.options as ProviderOptions | undefined, + credentials_configured: !!providerDoc.credentials?.encrypted_data, + decrypted_credentials: decrypted, + created_at: providerDoc.created_at, + updated_at: providerDoc.updated_at, + }; + } + + /** + * Helper to retrieve cached credentials or decrypt and cache them + */ + private static async getOrDecryptCredentials( + id: string, + encryptedCredentials?: Record | null + ): Promise> { + let credentials = ProviderManagerService.getCachedCredentials(id); + if (!credentials && encryptedCredentials) { + credentials = await decryptCredentials(encryptedCredentials as unknown as encrypted_credentials); + ProviderManagerService.cacheCredentials(id, credentials); + } + return credentials || {}; + } + + /** + * Helper to instantiate and optionally initialize a provider instance + */ + private static async createAndInitProvider( + id: string, + pluginName: string, + credentials: Record, + options: ProviderOptions, + initialize = true + ): Promise<{ providerInstance: SimpleNSProvider; healthy: boolean }> { + const providerInstance = await importAndInstantiateProvider(pluginName); + let healthy = true; + + if (initialize) { + const config: ProviderConfig = { + id, + credentials, + options, + }; + await providerInstance.initialize(config); + healthy = await providerInstance.healthCheck(); + } + + return { providerInstance, healthy }; + } + + /** + * Test live provider connection / credentials + */ + public static async testProviderConnection(data: { + provider_id?: string; + plugin_name?: string; + credentials?: Record; + options?: ProviderOptions; + }): Promise<{ success: boolean; message: string }> { + let pluginName = data.plugin_name; + let credentials = data.credentials; + let options: ProviderOptions = data.options || {}; + + if (data.provider_id) { + const existing = await Provider.findOne({ id: data.provider_id }); + if (!existing) { + throw new Error(`Provider '${data.provider_id}' not found.`); + } + pluginName = existing.plugin_name; + options = { ...(existing.options as ProviderOptions), ...options }; + if (!credentials || Object.keys(credentials).length === 0) { + credentials = await ProviderManagerService.getOrDecryptCredentials(data.provider_id, existing.credentials); + } + } + + if (!pluginName) { + throw new Error('plugin_name or provider_id is required.'); + } + if (!credentials) { + throw new Error('Provider credentials are required.'); + } + + try { + const { healthy } = await ProviderManagerService.createAndInitProvider( + data.provider_id || 'test-connection', + pluginName, + credentials, + options, + true + ); + + if (healthy) { + return { success: true, message: 'Provider connection and health check succeeded.' }; + } else { + return { success: false, message: 'Provider health check failed.' }; + } + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + return { success: false, message: `Connection test error: ${message}` }; + } + } + + /** + * Load, decrypt credentials, instantiate and register provider into PluginRegistry + */ + public static async loadAndRegisterProvider( + providerDoc: provider_document, + initialize = true + ): Promise { + const credentials = await ProviderManagerService.getOrDecryptCredentials(providerDoc.id, providerDoc.credentials); + const { providerInstance, healthy } = await ProviderManagerService.createAndInitProvider( + providerDoc.id, + providerDoc.plugin_name, + credentials, + (providerDoc.options as ProviderOptions) || {}, + initialize + ); + + if (initialize && !healthy) { + logger.warn(`Provider '${providerDoc.id}' failed health check on initial load.`); + } + + PluginRegistry.registerOrReplace( + providerInstance, + providerDoc.id, + providerDoc.options?.priority ?? 0 + ); + logger.success(`Registered provider '${providerDoc.id}' in PluginRegistry.`); + } + + /** + * Register remote synchronization handlers with Redis Pub/Sub + */ + public static registerSyncHandlers(): void { + PluginSyncService.on('PROVIDER_UPSERTED', async (msg) => { + const { provider_id } = msg.payload; + if (!provider_id) return; + + logger.info(`Sync: Handling remote PROVIDER_UPSERTED for '${provider_id}'...`); + ProviderManagerService.invalidateCredentialCache(provider_id); + const providerDoc = await Provider.findOne({ id: provider_id }); + + if (providerDoc && providerDoc.enabled) { + try { + await ProviderManagerService.loadAndRegisterProvider(providerDoc.toObject() as provider_document); + } catch (err) { + logger.error(`Failed to reload synced provider '${provider_id}':`, err); + } + } else { + PluginRegistry.unregister(provider_id); + } + }); + + PluginSyncService.on('PROVIDER_DELETED', async (msg) => { + const { provider_id } = msg.payload; + if (!provider_id) return; + + logger.info(`Sync: Handling remote PROVIDER_DELETED for '${provider_id}'...`); + ProviderManagerService.invalidateCredentialCache(provider_id); + PluginRegistry.unregister(provider_id); + }); + } +} diff --git a/src/plugins/sync/plugin-sync.service.ts b/src/plugins/sync/plugin-sync.service.ts new file mode 100644 index 0000000..70d41fa --- /dev/null +++ b/src/plugins/sync/plugin-sync.service.ts @@ -0,0 +1,182 @@ +/** + * Plugin System Redis Pub/Sub Synchronization Service + * + * Synchronizes plugin installations, provider changes, and channel routing + * across all running API and notification_processor containers in real time. + */ + +import { randomUUID } from 'crypto'; +import type { Redis as RedisType } from 'ioredis'; +import { getRedisClient, connectRedis } from '@src/config/redis.config.js'; +import { pluginLoaderLogger as logger } from '@src/workers/utils/logger.js'; +import type { PluginSyncAction, PluginSyncHandler, PluginSyncMessage, PluginSyncPayload } from '@src/types/types.js'; + +export const PLUGIN_SYNC_CHANNEL = 'simplens:plugin_system:sync'; + +export class PluginSyncServiceClass { + private static instance: PluginSyncServiceClass; + private instanceId: string; + private subClient: RedisType | null = null; + private isSubscribed = false; + private handlers: Map = new Map(); + private installMutex: Promise = Promise.resolve(); + + private constructor() { + this.instanceId = process.env.HOSTNAME || process.env.WORKER_ID || `instance-${randomUUID().slice(0, 8)}`; + } + + public static getInstance(): PluginSyncServiceClass { + if (!PluginSyncServiceClass.instance) { + PluginSyncServiceClass.instance = new PluginSyncServiceClass(); + } + return PluginSyncServiceClass.instance; + } + + /** + * Run an asynchronous task with in-process mutex locking + * Useful for protecting the .plugins/ folder during npm install/uninstall + */ + public async runWithLock(task: () => Promise): Promise { + let release: () => void = () => {}; + const nextLock = new Promise((resolve) => { + release = resolve; + }); + + const currentLock = this.installMutex; + this.installMutex = nextLock; + + await currentLock; + try { + return await task(); + } finally { + release(); + } + } + + /** + * Register an event handler for a specific action (or '*' for all actions) + */ + public on(action: PluginSyncAction | '*', handler: PluginSyncHandler): void { + const list = this.handlers.get(action) || []; + list.push(handler); + this.handlers.set(action, list); + } + + /** + * Publish an event to the Redis synchronization channel + */ + public async publish(action: PluginSyncAction, payload: PluginSyncPayload = {}): Promise { + try { + const redis = getRedisClient(); + const message: PluginSyncMessage = { + event_id: randomUUID(), + source_instance_id: this.instanceId, + timestamp: new Date().toISOString(), + action, + payload, + }; + + await redis.publish(PLUGIN_SYNC_CHANNEL, JSON.stringify(message)); + logger.info(`Published sync event [${action}] to ${PLUGIN_SYNC_CHANNEL}`); + } catch (err) { + logger.error(`Failed to publish sync event [${action}] to Redis`, err); + } + } + + /** + * Initialize and start the Redis subscriber + */ + public async startSubscriber(): Promise { + if (this.isSubscribed) { + return; + } + + try { + await connectRedis(); + const baseClient = getRedisClient(); + this.subClient = baseClient.duplicate(); + + this.subClient.on('error', (err: Error) => { + logger.error('Plugin sync Redis subscriber error:', err); + }); + + this.subClient.on('close', () => { + this.isSubscribed = false; + logger.warn('Plugin sync Redis subscriber connection closed. Reconnecting in 3s...'); + setTimeout(() => { + void this.startSubscriber(); + }, 3000); + }); + + if (this.subClient.status !== 'ready' && this.subClient.status !== 'connecting') { + await this.subClient.connect(); + } + await this.subClient.subscribe(PLUGIN_SYNC_CHANNEL); + this.isSubscribed = true; + + this.subClient.on('message', async (channel, messageStr) => { + if (channel !== PLUGIN_SYNC_CHANNEL) return; + + try { + const message: PluginSyncMessage = JSON.parse(messageStr); + + // Skip events emitted by this instance to prevent loops + if (message.source_instance_id === this.instanceId) { + logger.debug(`Ignoring sync event [${message.action}] from self (${this.instanceId})`); + return; + } + + logger.info(`Received sync event [${message.action}] from ${message.source_instance_id}`); + + // Execute action-specific handlers + const specificHandlers = this.handlers.get(message.action) || []; + for (const handler of specificHandlers) { + try { + await handler(message); + } catch (handlerErr) { + logger.error(`Error in sync handler for [${message.action}]:`, handlerErr); + } + } + + // Execute wildcard handlers + const wildcardHandlers = this.handlers.get('*') || []; + for (const handler of wildcardHandlers) { + try { + await handler(message); + } catch (handlerErr) { + logger.error(`Error in wildcard sync handler:`, handlerErr); + } + } + } catch (parseErr) { + logger.error('Failed to parse sync message from Redis:', parseErr); + } + }); + + logger.success(`Subscribed to plugin sync channel: ${PLUGIN_SYNC_CHANNEL}`); + } catch (err) { + logger.error('Failed to initialize Redis subscriber for plugin sync:', err); + setTimeout(() => { + void this.startSubscriber(); + }, 3000); + } + } + + /** + * Disconnect the subscriber + */ + public async stopSubscriber(): Promise { + if (this.subClient) { + try { + await this.subClient.unsubscribe(PLUGIN_SYNC_CHANNEL); + await this.subClient.quit(); + } catch (err) { + logger.error('Error stopping plugin sync subscriber:', err); + } finally { + this.subClient = null; + this.isSubscribed = false; + } + } + } +} + +export const PluginSyncService = PluginSyncServiceClass.getInstance(); diff --git a/src/processors/delayed/delayed.consumer.ts b/src/processors/delayed/delayed.consumer.ts index f6c42bc..8cdd714 100644 --- a/src/processors/delayed/delayed.consumer.ts +++ b/src/processors/delayed/delayed.consumer.ts @@ -3,25 +3,24 @@ * Channel-agnostic - routes based on dynamic target_topic */ -import { Consumer, EachMessagePayload } from 'kafkajs'; +import { EachMessagePayload } from 'kafkajs'; import { kafka } from '@src/config/kafka.config.js'; -import { CORE_TOPICS } from '@src/types/types.js'; +import { CORE_TOPICS, type KafkaConsumerState } from '@src/types/types.js'; import { delayedNotificationTopicSchema } from '@src/types/schemas.js'; import { addToDelayedQueue } from './delayed.queue.js'; import { delayedWorkerLogger as logger } from '@src/workers/utils/logger.js'; const CONSUMER_GROUP_ID = 'delayed-worker-group'; -interface ConsumerState { - consumer: Consumer | null; - isConsuming: boolean; -} - -const state: ConsumerState = { +const state: KafkaConsumerState = { consumer: null, isConsuming: false }; +let lastHeartbeat = Date.now(); +let hasCrashed = false; +let crashReason: string | undefined = undefined; + /** * Process a delayed notification message * Returns true if message was successfully added to queue @@ -76,6 +75,40 @@ export const startDelayedConsumer = async (): Promise => { await state.consumer.connect(); logger.success('Delayed consumer connected'); + lastHeartbeat = Date.now(); + hasCrashed = false; + crashReason = undefined; + + if (typeof state.consumer.on === 'function' && state.consumer.events) { + state.consumer.on(state.consumer.events.HEARTBEAT, () => { + lastHeartbeat = Date.now(); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + state.consumer.on(state.consumer.events.CRASH, (event: any) => { + hasCrashed = true; + state.isConsuming = false; + const crashError = event?.payload?.error; + crashReason = crashError ? String(crashError.message || crashError) : 'Unknown crash'; + logger.error(`Delayed consumer crashed: ${crashReason}`); + }); + + state.consumer.on(state.consumer.events.STOP, () => { + state.isConsuming = false; + logger.warn('Delayed consumer stopped event received'); + }); + + state.consumer.on(state.consumer.events.DISCONNECT, () => { + state.isConsuming = false; + logger.warn('Delayed consumer disconnected'); + }); + + state.consumer.on(state.consumer.events.CONNECT, () => { + state.isConsuming = true; + lastHeartbeat = Date.now(); + }); + } + await state.consumer.subscribe({ topic: CORE_TOPICS.delayed_notification, fromBeginning: false @@ -87,6 +120,7 @@ export const startDelayedConsumer = async (): Promise => { await state.consumer.run({ autoCommit: false, eachMessage: async (payload) => { + lastHeartbeat = Date.now(); try { const success = await processDelayedMessage(payload); @@ -122,6 +156,7 @@ export const stopDelayedConsumer = async (): Promise => { } state.isConsuming = false; + hasCrashed = false; try { await state.consumer.stop(); @@ -140,3 +175,14 @@ export const stopDelayedConsumer = async (): Promise => { export const isConsumerActive = (): boolean => { return state.isConsuming && state.consumer !== null; }; + +/** + * Check if delayed consumer is healthy (running, not crashed, heartbeat fresh) + */ +export const isDelayedConsumerHealthy = (maxHeartbeatStalenessMs = 90000): boolean => { + if (!state.consumer || !state.isConsuming || hasCrashed) { + return false; + } + const isStale = (Date.now() - lastHeartbeat) > maxHeartbeatStalenessMs; + return !isStale; +}; diff --git a/src/processors/delayed/delayed.poller.ts b/src/processors/delayed/delayed.poller.ts index 3697426..e975a4d 100644 --- a/src/processors/delayed/delayed.poller.ts +++ b/src/processors/delayed/delayed.poller.ts @@ -9,12 +9,14 @@ */ import { env } from '@src/config/env.config.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { claimDueEvents, confirmProcessed, reAddToQueue, getDueEventCount, releaseClaim } from './delayed.queue.js'; import { publishToTarget } from './target.producer.js'; import { publishDLQFailureStatus } from './dlq.status.js'; import { delayedWorkerLogger as logger } from '@src/workers/utils/logger.js'; -import type { delayed_notification_topic } from '@src/types/types.js'; +import type { DelayedEventWithRetries } from '@src/types/types.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; +import { calculateExponentialBackoff } from '@src/utils/backoff.utils.js'; let pollerInterval: NodeJS.Timeout | null = null; let isPolling = false; @@ -22,19 +24,13 @@ let isPolling = false; /** * Extended event type with poller retry tracking */ -interface DelayedEventWithRetries extends delayed_notification_topic { - _pollerRetries?: number; -} - /** * Calculate exponential backoff delay * @param retryCount Current retry count * @returns Delay in milliseconds (5s, 10s, 20s, 40s, capped at 60s) */ const calculateBackoff = (retryCount: number): number => { - const baseDelay = 5000; // 5 seconds - const maxDelay = 60000; // 60 seconds - return Math.min(baseDelay * Math.pow(2, retryCount), maxDelay); + return calculateExponentialBackoff(retryCount, 5000, 60000); }; /** @@ -182,6 +178,15 @@ export const startDelayedPoller = (): void => { logger.success('Delayed poller started'); }; +// Listen for dynamic configuration updates and hot-adjust poller interval +dynamicConfig.on('change', () => { + if (pollerInterval) { + clearInterval(pollerInterval); + pollerInterval = setInterval(pollForDueEvents, env.DELAYED_POLL_INTERVAL_MS); + logger.info(`Adjusted delayed poller interval to ${env.DELAYED_POLL_INTERVAL_MS}ms`); + } +}); + /** * Stop the delayed poller */ diff --git a/src/processors/delayed/delayed.processor.ts b/src/processors/delayed/delayed.processor.ts index 402e554..13f1490 100644 --- a/src/processors/delayed/delayed.processor.ts +++ b/src/processors/delayed/delayed.processor.ts @@ -10,19 +10,27 @@ * - Atomic Lua script prevents duplicate processing across multiple workers */ +import mongoose from 'mongoose'; +import { connectMongoDB } from '@src/config/db.config.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { connectRedis, disconnectRedis } from '@src/config/redis.config.js'; -import { initTargetProducer, disconnectTargetProducer } from './target.producer.js'; -import { initDLQStatusProducer, disconnectDLQStatusProducer } from './dlq.status.js'; -import { startDelayedConsumer, stopDelayedConsumer } from './delayed.consumer.js'; -import { startDelayedPoller, stopDelayedPoller } from './delayed.poller.js'; +import { initTargetProducer, disconnectTargetProducer, isTargetProducerActive } from './target.producer.js'; +import { initDLQStatusProducer, disconnectDLQStatusProducer, isDLQProducerActive } from './dlq.status.js'; +import { startDelayedConsumer, stopDelayedConsumer, isDelayedConsumerHealthy } from './delayed.consumer.js'; +import { startDelayedPoller, stopDelayedPoller, isPollerActive } from './delayed.poller.js'; import { delayedWorkerLogger as logger } from '@src/workers/utils/logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; +import { getRedisClient } from '@src/config/redis.config.js'; +import { createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeServer } from '@src/types/types.js'; //Import the admin channel provider files here for them to self-register import "@src/admin-alerts/channels/discord.channel.js"; import "@src/admin-alerts/channels/telegram.channel.js"; let isShuttingDown = false; +let dbConnection: Awaited> | null = null; +let probeServer: HealthProbeServer | null = null; /** * Graceful shutdown handler @@ -56,6 +64,17 @@ const gracefulShutdown = async (signal: string): Promise => { logger.info('Disconnecting Redis...'); await disconnectRedis(); + // 5. Disconnect MongoDB + if (dbConnection) { + logger.info('Disconnecting MongoDB...'); + await dbConnection.disconnect(); + } + + // 6. Stop health probe server + if (probeServer) { + await probeServer.stop(); + } + logger.success('Graceful shutdown complete'); process.exit(0); } catch (err) { @@ -105,7 +124,13 @@ const main = async (): Promise => { logger.info('Starting Delayed Processor...'); try { - // 1. Connect to Redis (for delayed queue) + // 1. Connect to MongoDB & Initialize Dynamic Configuration + logger.info('Connecting to MongoDB...'); + dbConnection = await connectMongoDB(); + logger.success('Connected to MongoDB'); + await dynamicConfig.initialize(false); + + // 2. Connect to Redis (for delayed queue) logger.info('Connecting to Redis...'); await connectRedis(); @@ -128,6 +153,55 @@ const main = async (): Promise => { logger.success('Delayed Processor is running!'); logger.info('================================'); + // 5. Start health probe server for Kubernetes + probeServer = createHealthProbeServer({ + serviceName: 'delayed-processor', + readinessChecks: [ + { + name: 'mongodb', + check: () => !isShuttingDown && mongoose.connection.readyState === 1 + }, + { + name: 'redis', + check: () => { + if (isShuttingDown) return false; + try { + return getRedisClient().status === 'ready'; + } catch { + return false; + } + } + }, + { + name: 'delayed_consumer', + check: () => !isShuttingDown && isDelayedConsumerHealthy() + }, + { + name: 'delayed_poller', + check: () => !isShuttingDown && isPollerActive() + }, + { + name: 'kafka_producers', + check: () => !isShuttingDown && isTargetProducerActive() && isDLQProducerActive() + } + ], + livenessChecks: [ + { + name: 'process', + check: () => !isShuttingDown + }, + { + name: 'delayed_consumer', + check: () => !isShuttingDown && isDelayedConsumerHealthy() + }, + { + name: 'delayed_poller', + check: () => !isShuttingDown && isPollerActive() + } + ] + }); + await probeServer.start(); + // Register shutdown handlers registerShutdownHandlers(); } catch (err) { diff --git a/src/processors/delayed/dlq.status.ts b/src/processors/delayed/dlq.status.ts index 42ec57b..01f8fe9 100644 --- a/src/processors/delayed/dlq.status.ts +++ b/src/processors/delayed/dlq.status.ts @@ -87,3 +87,10 @@ export const disconnectDLQStatusProducer = async (): Promise => { logger.info('DLQ status producer disconnected'); } }; + +/** + * Check if DLQ status producer is active + */ +export const isDLQProducerActive = (): boolean => { + return producer !== null; +}; diff --git a/src/processors/delayed/target.producer.ts b/src/processors/delayed/target.producer.ts index 51235fa..86fce9f 100644 --- a/src/processors/delayed/target.producer.ts +++ b/src/processors/delayed/target.producer.ts @@ -63,3 +63,10 @@ export const disconnectTargetProducer = async (): Promise => { logger.info('Target producer disconnected'); } }; + +/** + * Check if target producer is active + */ +export const isTargetProducerActive = (): boolean => { + return producer !== null; +}; diff --git a/src/processors/shared/delayed.producer.ts b/src/processors/shared/delayed.producer.ts index 584c898..4ef2865 100644 --- a/src/processors/shared/delayed.producer.ts +++ b/src/processors/shared/delayed.producer.ts @@ -6,6 +6,8 @@ import { Producer, Partitioners } from 'kafkajs'; import { kafka } from '@src/config/kafka.config.js'; import { CORE_TOPICS, getTopicForChannel, type delayed_notification_topic } from '@src/types/types.js'; +import { calculateExponentialBackoff } from '@src/utils/backoff.utils.js'; + let producer: Producer | null = null; /** @@ -28,9 +30,7 @@ export const initDelayedProducer = async (): Promise => { * Calculate exponential backoff delay in milliseconds */ const calculateBackoffDelay = (retryCount: number): number => { - const baseDelay = 1000; // 1 second - const maxDelay = 300000; // 5 minutes - return Math.min(baseDelay * Math.pow(2, retryCount), maxDelay); + return calculateExponentialBackoff(retryCount, 1000, 300000); }; /** @@ -96,3 +96,10 @@ export const disconnectDelayedProducer = async (): Promise => { producer = null; } }; + +/** + * Check if delayed producer is active + */ +export const isDelayedProducerActive = (): boolean => { + return producer !== null; +}; diff --git a/src/processors/shared/idempotency.ts b/src/processors/shared/idempotency.ts index 3064dfd..9f2ab03 100644 --- a/src/processors/shared/idempotency.ts +++ b/src/processors/shared/idempotency.ts @@ -6,17 +6,12 @@ import { getRedisClient } from '@src/config/redis.config.js'; import { env } from '@src/config/env.config.js'; +import type { IdempotencyRecord } from '@src/types/types.js'; // Redis key prefix for idempotency const IDEMPOTENCY_PREFIX = 'idempotency'; // Idempotency record stored in Redis -interface IdempotencyRecord { - status: 'processing' | 'delivered' | 'failed' | 'rate_limited'; - retry_count: number; - updated_at: string; -} - /** * Build Redis key for idempotency check */ diff --git a/src/processors/shared/rate-limiter.ts b/src/processors/shared/rate-limiter.ts index 05e2fbb..3c4b3aa 100644 --- a/src/processors/shared/rate-limiter.ts +++ b/src/processors/shared/rate-limiter.ts @@ -6,22 +6,19 @@ */ import { getRedisClient } from '@src/config/redis.config.js'; +import { env } from '@src/config/env.config.js'; import { getRateLimitConfig as getPluginRateLimitConfig } from '@src/plugins/index.js'; +import { PluginRegistry } from '@src/plugins/loader/registry.js'; +import Provider from '@src/database/models/provider.models.js'; import { rateLimiterLogger as logger } from '@src/workers/utils/logger.js'; - -// Redis key prefixes -const TOKENS_KEY_PREFIX = 'ratelimit:tokens'; -const LAST_REFILL_KEY_PREFIX = 'ratelimit:last_refill'; -const QUEUE_POSITION_KEY_PREFIX = 'ratelimit:queue_position'; - -/** Time interval for refill rate */ -type RefillInterval = 'second' | 'minute' | 'hour' | 'day'; - -interface RateLimitConfig { - maxTokens: number; - refillRate: number; - refillInterval?: RefillInterval; -} +import type { + RateLimitConfig, + RefillInterval, + RateLimitResult, + ProviderRateLimitStatus, + ProviderRateLimitListResponse, + ProviderBasicMeta, +} from '@src/types/types.js'; // Conversion divisors to convert rate to per-second const INTERVAL_TO_SECONDS: Record = { @@ -35,14 +32,14 @@ const INTERVAL_TO_SECONDS: Record = { * Normalize refill rate to tokens per second * Converts interval-based rates (e.g., 500/day) to per-second rates */ -function normalizeRefillRate(config: RateLimitConfig): number { +export function normalizeRefillRate(config: RateLimitConfig): number { const interval = config.refillInterval || 'second'; const divisor = INTERVAL_TO_SECONDS[interval] || 1; return config.refillRate / divisor; } // Default rate limit config -const DEFAULT_RATE_LIMIT: RateLimitConfig = { +export const DEFAULT_RATE_LIMIT: RateLimitConfig = { maxTokens: 100, refillRate: 10, refillInterval: 'second', @@ -52,7 +49,7 @@ const DEFAULT_RATE_LIMIT: RateLimitConfig = { * Get rate limit configuration for a provider * Priority: Plugin config > Default */ -const getConfig = (providerId: string): RateLimitConfig => { +export const getConfig = (providerId: string): RateLimitConfig => { // Try plugin registry first const pluginConfig = getPluginRateLimitConfig(providerId); if (pluginConfig) { @@ -64,49 +61,64 @@ const getConfig = (providerId: string): RateLimitConfig => { }; /** - * Build Redis keys for a provider + * Build Redis keys for a provider using Redis Cluster hash tags {providerId} */ -const buildKeys = (providerId: string): { tokensKey: string; lastRefillKey: string; queueKey: string } => { +export const buildKeys = (providerId: string): { + tokensKey: string; + lastRefillKey: string; + queueKey: string; + exhaustedCountKey: string; + lastExhaustedKey: string; +} => { return { - tokensKey: `${TOKENS_KEY_PREFIX}:${providerId}`, - lastRefillKey: `${LAST_REFILL_KEY_PREFIX}:${providerId}`, - queueKey: `${QUEUE_POSITION_KEY_PREFIX}:${providerId}` + tokensKey: `ratelimit:{${providerId}}:tokens`, + lastRefillKey: `ratelimit:{${providerId}}:last_refill`, + queueKey: `ratelimit:{${providerId}}:queue_position`, + exhaustedCountKey: `ratelimit:{${providerId}}:exhausted_count`, + lastExhaustedKey: `ratelimit:{${providerId}}:last_exhausted`, }; }; +// In-memory status response cache to debounce high-frequency dashboard queries +let cachedStatusResponse: { data: ProviderRateLimitListResponse; timestamp: number } | null = null; +const CACHE_TTL_MS = 1000; // 1 second + /** - * Token Bucket Algorithm Result + * Clear the in-memory rate limit status cache */ -export interface RateLimitResult { - allowed: boolean; - remainingTokens: number; - retryAfterMs?: number; - queuePosition?: number; // Position in queue when rate-limited -} +export const clearRateLimitStatusCache = (): void => { + cachedStatusResponse = null; +}; /** * Try to consume a token from the bucket - * Uses Redis Lua script for atomic operation with queue-based staggering + * Uses Redis Lua script for atomic operation with fixed retry delay */ export const consumeToken = async (providerId: string): Promise => { const redis = getRedisClient(); const config = getConfig(providerId); const normalizedRate = normalizeRefillRate(config); - const { tokensKey, lastRefillKey, queueKey } = buildKeys(providerId); + const { tokensKey, lastRefillKey, exhaustedCountKey, lastExhaustedKey } = buildKeys(providerId); + const fixedRetryDelay = env.RATE_LIMIT_RETRY_DELAY_MS; // Debug logging - logger.debug(`Provider: ${providerId}, Config: maxTokens=${config.maxTokens}, refillRate=${config.refillRate}/${config.refillInterval || 'second'} (normalized: ${normalizedRate.toFixed(6)}/sec)`); + logger.debug(`Provider: ${providerId}, Config: maxTokens=${config.maxTokens}, refillRate=${config.refillRate}/${config.refillInterval || 'second'} (normalized: ${normalizedRate.toFixed(6)}/sec), retryDelay=${fixedRetryDelay}ms`); const now = Date.now(); + // Rolling TTL: 7 days or at least 2x the duration needed to refill from 0 to full capacity + const ttlSeconds = Math.max(86400 * 7, Math.ceil((config.maxTokens / Math.max(normalizedRate, 0.0001)) * 2)); - // Lua script for atomic token bucket operation with queue position tracking + // Lua script for atomic token bucket operation with cluster-safe hash tag keys & telemetry const luaScript = ` local tokens_key = KEYS[1] local last_refill_key = KEYS[2] - local queue_key = KEYS[3] + local exhausted_count_key = KEYS[3] + local last_exhausted_key = KEYS[4] local max_tokens = tonumber(ARGV[1]) local refill_rate = tonumber(ARGV[2]) local now = tonumber(ARGV[3]) + local fixed_retry_delay = tonumber(ARGV[4]) + local ttl_seconds = tonumber(ARGV[5]) -- Get current state local current_tokens = tonumber(redis.call('GET', tokens_key)) or max_tokens @@ -120,43 +132,37 @@ export const consumeToken = async (providerId: string): Promise -- Try to consume a token if new_tokens >= 1 then new_tokens = new_tokens - 1 - redis.call('SET', tokens_key, new_tokens) - redis.call('SET', last_refill_key, now) - -- Reset queue position when tokens are available (batch consumed) - redis.call('SET', queue_key, 0) - redis.call('EXPIRE', queue_key, 86400) - return { 1, new_tokens, 0, 0 } -- allowed, remaining, wait_time, queue_position + redis.call('SET', tokens_key, new_tokens, 'EX', ttl_seconds) + redis.call('SET', last_refill_key, now, 'EX', ttl_seconds) + return { 1, new_tokens, 0 } else - -- Rate limited: calculate staggered delay based on queue position - local time_per_token = 1000 / refill_rate -- ms per token - local queue_position = redis.call('INCR', queue_key) - 1 - redis.call('EXPIRE', queue_key, 86400) -- 24h TTL - - -- Staggered delay: (position + 1) × time_per_token - local staggered_delay = (queue_position + 1) * time_per_token - - return { 0, new_tokens, staggered_delay, queue_position } -- denied, remaining, wait_time, queue_position + redis.call('INCR', exhausted_count_key) + redis.call('EXPIRE', exhausted_count_key, ttl_seconds) + redis.call('SET', last_exhausted_key, now, 'EX', ttl_seconds) + return { 0, new_tokens, fixed_retry_delay } end `; const result = await redis.eval( luaScript, - 3, // 3 keys now + 4, tokensKey, lastRefillKey, - queueKey, + exhaustedCountKey, + lastExhaustedKey, config.maxTokens.toString(), normalizedRate.toString(), - now.toString() - ) as [number, number, number, number]; + now.toString(), + fixedRetryDelay.toString(), + ttlSeconds.toString() + ) as [number, number, number]; - const [allowed, remainingTokens, retryAfterMs, queuePosition] = result; + const [allowed, remainingTokens, retryAfterMs] = result; return { allowed: allowed === 1, remainingTokens: Math.floor(remainingTokens), retryAfterMs: retryAfterMs > 0 ? Math.ceil(retryAfterMs) : undefined, - queuePosition: allowed === 0 ? queuePosition : undefined }; }; @@ -174,7 +180,7 @@ export const getTokenCount = async (providerId: string): Promise => { const [tokensStr, lastRefillStr] = await redis.mget(tokensKey, lastRefillKey); const currentTokens = tokensStr ? parseFloat(tokensStr) : config.maxTokens; - const lastRefill = lastRefillStr ? parseInt(lastRefillStr) : now; + const lastRefill = lastRefillStr ? parseInt(lastRefillStr, 10) : now; const elapsedSeconds = (now - lastRefill) / 1000; const tokensToAdd = elapsedSeconds * normalizedRate; @@ -183,11 +189,199 @@ export const getTokenCount = async (providerId: string): Promise => { }; /** - * Reset rate limiter for a provider (for testing) + * Fetch all provider rate limit statuses at scale using batched Redis MGET + */ +export const getAllProvidersRateLimitStatus = async (): Promise => { + const now = Date.now(); + + // Check in-memory cache to debounce frequent dashboard requests + if (cachedStatusResponse && (now - cachedStatusResponse.timestamp) < CACHE_TTL_MS) { + return cachedStatusResponse.data; + } + + const providerMap = new Map(); + + // 1. Gather providers from Database (primary) + try { + const dbProviders = await Provider.find().sort({ channel: 1, 'options.priority': -1 }).lean(); + for (const p of dbProviders) { + providerMap.set(p.id, { + id: p.id, + channel: p.channel, + plugin_name: p.plugin_name, + display_name: p.id, + enabled: p.enabled !== false, + }); + } + } catch { + // If MongoDB query fails or is not connected, continue to registry fallback + } + + // 2. Supplement/fallback from PluginRegistry + try { + const registered = PluginRegistry.getAllRegistered(); + for (const r of registered) { + if (!providerMap.has(r.id)) { + providerMap.set(r.id, { + id: r.id, + channel: r.provider.manifest.channel, + plugin_name: r.provider.manifest.name, + display_name: r.provider.manifest.displayName || r.id, + enabled: true, + }); + } else { + const existing = providerMap.get(r.id)!; + if (r.provider.manifest.displayName) { + existing.display_name = r.provider.manifest.displayName; + } + } + } + } catch { + // Registry check safe fallback + } + + const providersList = Array.from(providerMap.values()); + + if (providersList.length === 0) { + const emptyResponse: ProviderRateLimitListResponse = { + providers: [], + summary: { + total_providers: 0, + exhausted_providers: 0, + healthy_providers: 0, + }, + }; + cachedStatusResponse = { data: emptyResponse, timestamp: now }; + return emptyResponse; + } + + // 3. Build batched Redis keys for single MGET round-trip + const redis = getRedisClient(); + const allKeys: string[] = []; + + for (const p of providersList) { + const { tokensKey, lastRefillKey, exhaustedCountKey, lastExhaustedKey } = buildKeys(p.id); + allKeys.push(tokensKey, lastRefillKey, exhaustedCountKey, lastExhaustedKey); + } + + let mgetResults: (string | null)[]; + try { + mgetResults = await redis.mget(...allKeys); + } catch (redisErr) { + logger.error('Failed to batch query Redis for provider rate limits:', redisErr); + mgetResults = new Array(allKeys.length).fill(null); + } + + // 4. Derive metrics in application memory (O(N) with no write contention) + const providersStatus: ProviderRateLimitStatus[] = []; + let exhaustedCount = 0; + + for (let i = 0; i < providersList.length; i++) { + const p = providersList[i]; + const config = getConfig(p.id); + const normalizedRate = normalizeRefillRate(config); + const maxTokens = config.maxTokens; + + const tokensStr = mgetResults[i * 4]; + const lastRefillStr = mgetResults[i * 4 + 1]; + const exhaustedCountStr = mgetResults[i * 4 + 2]; + const lastExhaustedStr = mgetResults[i * 4 + 3]; + + const currentTokens = tokensStr !== null && tokensStr !== undefined ? parseFloat(tokensStr) : maxTokens; + const lastRefill = lastRefillStr ? parseInt(lastRefillStr, 10) : now; + const elapsedSeconds = Math.max(0, (now - lastRefill) / 1000); + const tokensToAdd = elapsedSeconds * normalizedRate; + const rawRemaining = Math.min(maxTokens, currentTokens + tokensToAdd); + + const remainingTokens = Math.min(maxTokens, Math.max(0, Math.round(rawRemaining * 10) / 10)); + const usedTokens = Math.max(0, Math.round((maxTokens - remainingTokens) * 10) / 10); + const usagePercentage = maxTokens > 0 ? Math.min(100, Math.max(0, Math.round((usedTokens / maxTokens) * 100))) : 0; + const isExhausted = remainingTokens < 1; + + if (isExhausted) { + exhaustedCount++; + } + + let resetsInMs = 0; + if (isExhausted && normalizedRate > 0) { + const tokensNeeded = 1 - remainingTokens; + resetsInMs = Math.ceil((tokensNeeded / normalizedRate) * 1000); + } + + let fullRefillInMs = 0; + if (remainingTokens < maxTokens && normalizedRate > 0) { + const tokensNeeded = maxTokens - remainingTokens; + fullRefillInMs = Math.ceil((tokensNeeded / normalizedRate) * 1000); + } + + const lastRefillAt = lastRefillStr ? new Date(parseInt(lastRefillStr, 10)) : null; + const lastExhaustedAt = lastExhaustedStr ? new Date(parseInt(lastExhaustedStr, 10)) : null; + const providerExhaustedCount = exhaustedCountStr ? parseInt(exhaustedCountStr, 10) : 0; + + providersStatus.push({ + provider_id: p.id, + channel: p.channel, + plugin_name: p.plugin_name, + display_name: p.display_name, + enabled: p.enabled, + config: { + max_tokens: maxTokens, + refill_rate: config.refillRate, + refill_interval: config.refillInterval || 'second', + normalized_refill_rate: normalizedRate, + }, + status: { + remaining_tokens: remainingTokens, + used_tokens: usedTokens, + usage_percentage: usagePercentage, + is_exhausted: isExhausted, + resets_in_ms: resetsInMs, + full_refill_in_ms: fullRefillInMs, + last_refill_at: lastRefillAt, + last_exhausted_at: lastExhaustedAt, + exhausted_count: providerExhaustedCount, + }, + }); + } + + const response: ProviderRateLimitListResponse = { + providers: providersStatus, + summary: { + total_providers: providersStatus.length, + exhausted_providers: exhaustedCount, + healthy_providers: providersStatus.length - exhaustedCount, + }, + }; + + cachedStatusResponse = { data: response, timestamp: now }; + return response; +}; + +/** + * Get rate limit status for a single provider + */ +export const getProviderRateLimitStatus = async (providerId: string): Promise => { + const all = await getAllProvidersRateLimitStatus(); + return all.providers.find((p) => p.provider_id === providerId) || null; +}; + +/** + * Reset rate limiter for a provider (refills bucket to capacity and wipes telemetry) */ export const resetRateLimiter = async (providerId: string): Promise => { const redis = getRedisClient(); - const { tokensKey, lastRefillKey, queueKey } = buildKeys(providerId); + const { tokensKey, lastRefillKey, queueKey, exhaustedCountKey, lastExhaustedKey } = buildKeys(providerId); + + await redis.del( + tokensKey, + lastRefillKey, + queueKey, + exhaustedCountKey, + lastExhaustedKey, + `ratelimit:tokens:${providerId}`, + `ratelimit:last_refill:${providerId}`, + `ratelimit:queue_position:${providerId}` + ); - await redis.del(tokensKey, lastRefillKey, queueKey); + clearRateLimitStatusCache(); }; diff --git a/src/processors/shared/status.producer.ts b/src/processors/shared/status.producer.ts index 2a1334c..47fe1b0 100644 --- a/src/processors/shared/status.producer.ts +++ b/src/processors/shared/status.producer.ts @@ -53,3 +53,10 @@ export const disconnectStatusProducer = async (): Promise => { producer = null; } }; + +/** + * Check if status producer is active + */ +export const isStatusProducerActive = (): boolean => { + return producer !== null; +}; diff --git a/src/processors/unified/unified.consumer.ts b/src/processors/unified/unified.consumer.ts index 42f3be4..096998e 100644 --- a/src/processors/unified/unified.consumer.ts +++ b/src/processors/unified/unified.consumer.ts @@ -10,8 +10,8 @@ * - Retry queue management */ -import { Consumer, EachMessagePayload } from 'kafkajs'; -import { kafka } from '@src/config/kafka.config.js'; +import { Consumer, EachMessagePayload, ConsumerCrashEvent } from 'kafkajs'; +import { kafka, ensureChannelTopic } from '@src/config/kafka.config.js'; import { NOTIFICATION_STATUS_SF } from '@src/types/types.js'; import { env } from '@src/config/env.config.js'; import { unifiedProcessorLogger as logger } from './unified.logger.js'; @@ -20,10 +20,18 @@ import { unifiedProcessorLogger as logger } from './unified.logger.js'; import { sendWithFallback, PluginRegistry, + getProviderCascade, resolveFallbackProviderId, validateNotificationForProvider } from '@src/plugins/index.js'; -import type { BaseNotification, DeliveryResult } from '@src/plugins/interfaces/provider.types.js'; +import type { + BaseNotification, + DeliveryResult, + RateLimitResult, + ConsumerHealthState, + ConsumerHealthCheckResult, + ConsumerHealthDetails +} from '@src/types/types.js'; // Shared utilities import { tryAcquireProcessingLock, setDelivered, setFailed, setRateLimited } from '@src/processors/shared/idempotency.js'; @@ -33,9 +41,14 @@ import { publishDelayed, buildDelayedPayloadGeneric } from '@src/processors/shar import { handleSchemaValidationFailure } from '../shared/schema-failure-handler.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; import status_outbox_model from '@src/database/models/status-outbox.models.js'; -// Track active consumers by channel + +// Re-export consumer health types +export type { ConsumerHealthState, ConsumerHealthCheckResult, ConsumerHealthDetails }; + +// State tracking for graceful shutdown & health const consumers: Map = new Map(); const consumingState: Map = new Map(); +const consumerHealthState: Map = new Map(); /** * Get Kafka topic name for a channel @@ -65,6 +78,8 @@ const publishSuccessStatus = async ( request_id: notification.request_id, client_id: notification.client_id, channel: channel, + provider: notification.provider, + provider_history: notification.provider_history, status: NOTIFICATION_STATUS_SF.delivered, message: messageId ? `Delivered via ${messageId}` : 'Notification sent successfully', retry_count: notification.retry_count, @@ -89,6 +104,8 @@ const publishFailureStatus = async ( request_id: notification.request_id, client_id: notification.client_id, channel: channel, + provider: notification.provider, + provider_history: notification.provider_history, status: NOTIFICATION_STATUS_SF.failed, message: errorMessage, retry_count: notification.retry_count, @@ -108,7 +125,8 @@ export const scheduleFallbackProviderHandoff = async ( channel: string, notification: BaseNotification, currentProviderId: string | undefined, - errorMessage: string + errorMessage: string, + delayMs?: number ): Promise => { const fallbackProviderId = resolveFallbackProviderId(channel, currentProviderId); @@ -134,17 +152,29 @@ export const scheduleFallbackProviderHandoff = async ( } await setFailed(notification.notification_id.toString(), notification.retry_count); - const delayedPayload = buildDelayedPayloadGeneric( - validationResult.data as unknown as Record, - channel, - 0 - ); + const delayedPayload = delayMs !== undefined + ? buildDelayedPayloadGeneric( + validationResult.data as unknown as Record, + channel, + 0, + delayMs + ) + : buildDelayedPayloadGeneric( + validationResult.data as unknown as Record, + channel, + 0 + ); // eslint-disable-next-line @typescript-eslint/no-explicit-any await publishDelayed(delayedPayload as any); + const isRateLimit = errorMessage.toLowerCase().includes('rate limit'); + const alertTitle = isRateLimit + ? `INFO: RATE LIMIT EXCEEDED, SWITCHING TO FALLBACK\n` + : `INFO: RETRY BUDGET EXHAUSTED, SWITCHING TO FALLBACK\n`; + void AdminAlertService.sendAlert( 'service_health', - `INFO: RETRY BUDGET EXHAUSTED, SWITCHING TO FALLBACK\n` + + alertTitle + `Notification ID: ${notification.notification_id}\n` + `Channel: ${channel}\n` + `Primary provider: ${currentProviderId || 'default'}\n` + @@ -155,8 +185,7 @@ export const scheduleFallbackProviderHandoff = async ( ); logger.warn( - `[${channel}] Retry budget exhausted for provider ${currentProviderId || 'default'}, ` + - `scheduled fallback handoff to ${fallbackProviderId}: ${notification.notification_id}` + `[${channel}] Fallback handoff scheduled for provider ${currentProviderId || 'default'} to ${fallbackProviderId}: ${notification.notification_id}` ); return true; }; @@ -247,6 +276,9 @@ const processMessage = async ( } const notification = validationResult.data as BaseNotification; + if (!notification.provider_history) { + notification.provider_history = []; + } const notificationId = notification.notification_id.toString(); logger.info(`[${channel}] Processing notification: ${notificationId} (retry: ${notification.retry_count})`); @@ -262,34 +294,59 @@ const processMessage = async ( logger.info(`[${channel}] Retrying previously failed: ${notificationId}`); } - // 4. Rate limit check - uses provider ID for per-provider rate limiting - const rateLimitResult = await consumeToken(providerId!); + // 4. Rate limit check across the provider cascade + const cascade = getProviderCascade(channel, rawData.provider); + let selectedProviderId: string | null = null; + let lastRateLimitResult: RateLimitResult | null = null; + + for (let i = 0; i < cascade.length; i++) { + const candidateId = cascade[i]; + const rateLimitResult = await consumeToken(candidateId); + + if (rateLimitResult.allowed) { + selectedProviderId = candidateId; + if (i > 0) { + logger.info(`[${channel}] Cascade failover: '${cascade[0]}' rate-limited, using fallback '${candidateId}': ${notificationId}`); + void AdminAlertService.sendAlert('service_health', + `ℹ️ RATE LIMIT CASCADE: USING FALLBACK (${i + 1}/${cascade.length})\n` + + `Channel: ${channel}\n` + + `Notification ID: ${notificationId}\n` + + `Rate-limited providers: ${cascade.slice(0, i).join(', ')}\n` + + `Active fallback: ${candidateId}\n` + + `Action: Check primary provider rate limits if this persists.`, + { severity: 'info', notificationId, channel }); + } + break; + } else { + lastRateLimitResult = rateLimitResult; + const retryDelay = rateLimitResult.retryAfterMs ?? env.RATE_LIMIT_RETRY_DELAY_MS; + notification.provider_history.push({ + provider: candidateId, + status: 'rate_limited', + error_code: 'RATE_LIMITED', + error_message: `Provider '${candidateId}' rate limit exceeded (retry after ${retryDelay}ms)`, + retryable: true, + attempted_at: new Date() + }); + logger.warn(`[${channel}] Provider '${candidateId}' rate limited (${i + 1}/${cascade.length}): ${notificationId}`); + } + } - if (!rateLimitResult.allowed) { - logger.warn(`[${channel}] Rate limited: ${notificationId}, retry after ${rateLimitResult.retryAfterMs}ms`); + if (!selectedProviderId) { + // All providers in cascade are rate-limited + logger.warn(`[${channel}] All providers in cascade rate-limited: ${notificationId}`); const newRetryCount = notification.retry_count + 1; if (newRetryCount > env.MAX_RETRY_COUNT) { const currentProviderId = notification.provider || PluginRegistry.getDefaultProviderId(channel); - const handoffScheduled = await scheduleFallbackProviderHandoff( - channel, - notification, - currentProviderId, - `Rate limited after ${env.MAX_RETRY_COUNT} retries` - ); - - if (handoffScheduled) { - return true; - } - - logger.error(`[${channel}] Max retries exceeded (rate limited): ${notificationId}`); + logger.error(`[${channel}] Max retries exceeded (all cascade providers rate limited): ${notificationId}`); void AdminAlertService.sendAlert('failed_notification', `❌ MAX RETRIES EXCEEDED (RATE LIMITED)\n` + `Notification ID: ${notificationId}\n` + `Channel: ${channel}\n` + `Provider: ${currentProviderId || providerId || 'default'}\n` + - `Root cause: Provider rate limit exhausted after ${env.MAX_RETRY_COUNT} retries\n` + + `Root cause: All cascade providers rate limit exhausted after ${env.MAX_RETRY_COUNT} retries\n` + `Action: Check provider rate limits in simplens.config.yaml. Consider increasing limits or adding fallback provider.`, { severity: 'critical', notificationId, channel }); @@ -298,23 +355,39 @@ const processMessage = async ( return true; } - // Push to delayed queue using rate limiter's retryAfterMs + // Push to delayed queue using fixed retry delay from rate limiter await setRateLimited(notificationId, validationResult.data.retry_count); + const retryDelay = lastRateLimitResult?.retryAfterMs ?? env.RATE_LIMIT_RETRY_DELAY_MS; const delayedPayload = buildDelayedPayloadGeneric( notification as unknown as Record, channel, newRetryCount, - rateLimitResult.retryAfterMs // Use exact delay from rate limiter + retryDelay ); // eslint-disable-next-line @typescript-eslint/no-explicit-any await publishDelayed(delayedPayload as any); - logger.info(`[${channel}] Rate limited, scheduled retry in ${rateLimitResult.retryAfterMs}ms: ${notificationId} (retry ${newRetryCount})`); + logger.info(`[${channel}] Rate limited, scheduled retry in ${retryDelay}ms: ${notificationId} (retry ${newRetryCount})`); return true; } + // If selected fallback provider differs from original provider, update notification provider + if (selectedProviderId !== notification.provider) { + notification.provider = selectedProviderId; + } + // 5. Send via plugin router (with auto-fallback) const result: DeliveryResult = await sendWithFallback(channel, notification); + // Merge router dispatch attempts into provider_history + if (result.attempts && result.attempts.length > 0) { + notification.provider_history.push(...result.attempts); + } + + // Update notification provider if result specifies the delivering/failing provider + if (result.provider && result.provider !== notification.provider) { + notification.provider = result.provider; + } + if (result.success) { // 6a. Success try { @@ -343,6 +416,8 @@ const processMessage = async ( await status_outbox_model.create({ notification_id: notificationId, status: NOTIFICATION_STATUS_SF.delivered, + provider: notification.provider, + provider_history: notification.provider_history, processed: false }); logger.info(`[${channel}] Created status_outbox fallback for: ${notificationId}`); @@ -448,6 +523,9 @@ export const startUnifiedConsumer = async (channel: string): Promise => { return; } + // Ensure the Kafka topic exists before attempting to subscribe + await ensureChannelTopic(channel); + const topic = getTopicForChannel(channel); const groupId = getConsumerGroupId(channel); @@ -459,6 +537,71 @@ export const startUnifiedConsumer = async (channel: string): Promise => { rebalanceTimeout: 60000, heartbeatInterval: 3000, }); + + const health: ConsumerHealthState = { + channel, + isRunning: false, + hasCrashed: false, + lastHeartbeat: Date.now(), + }; + consumerHealthState.set(channel, health); + + // Register Kafka lifecycle and crash event listeners + if (typeof consumer.on === 'function' && consumer.events) { + consumer.on(consumer.events.HEARTBEAT, () => { + const h = consumerHealthState.get(channel); + if (h) { + h.lastHeartbeat = Date.now(); + } + }); + + consumer.on(consumer.events.CRASH, async (event: ConsumerCrashEvent) => { + const h = consumerHealthState.get(channel); + const crashError = event?.payload?.error; + const errorMsg = crashError ? String(crashError.message || crashError) : 'Unknown crash'; + if (h) { + h.hasCrashed = true; + h.isRunning = false; + h.crashReason = errorMsg; + } + logger.error(`[${channel}] Consumer CRASHED: ${errorMsg}`, crashError); + + void AdminAlertService.sendAlert( + 'service_health', + `🔴 KAFKA CONSUMER CRASHED - Unified Processor\n` + + `Channel: ${channel}\n` + + `Group: ${groupId}\n` + + `Restartable: ${Boolean(event?.payload?.restart)}\n` + + `Error: ${errorMsg}\n` + + `Action: Kubernetes liveness probe will detect failure and restart container.`, + { severity: 'critical' } + ); + }); + + consumer.on(consumer.events.STOP, () => { + const h = consumerHealthState.get(channel); + if (h) { + h.isRunning = false; + } + logger.warn(`[${channel}] Consumer stopped event received`); + }); + + consumer.on(consumer.events.DISCONNECT, () => { + const h = consumerHealthState.get(channel); + if (h) { + h.isRunning = false; + } + logger.warn(`[${channel}] Consumer disconnected`); + }); + + consumer.on(consumer.events.CONNECT, () => { + const h = consumerHealthState.get(channel); + if (h) { + h.isRunning = true; + h.lastHeartbeat = Date.now(); + } + }); + } await consumer.connect(); await consumer.subscribe({ topic, fromBeginning: false }); @@ -473,6 +616,11 @@ export const startUnifiedConsumer = async (channel: string): Promise => { if (!consumingState.get(channel)) { return; // Consumer is stopping } + + const h = consumerHealthState.get(channel); + if (h) { + h.lastHeartbeat = Date.now(); + } const shouldCommit = await processMessage(channel, payload); @@ -491,6 +639,9 @@ export const startUnifiedConsumer = async (channel: string): Promise => { } }); + health.isRunning = true; + health.lastHeartbeat = Date.now(); + logger.success(`[${channel}] Consumer started`); }; @@ -505,12 +656,17 @@ export const stopUnifiedConsumer = async (channel: string): Promise => { logger.info(`[${channel}] Stopping consumer...`); consumingState.set(channel, false); + const health = consumerHealthState.get(channel); + if (health) { + health.isRunning = false; + } try { await consumer.stop(); await consumer.disconnect(); consumers.delete(channel); consumingState.delete(channel); + consumerHealthState.delete(channel); logger.info(`[${channel}] Consumer stopped`); } catch (err) { logger.error(`[${channel}] Error stopping consumer:`, err); @@ -526,3 +682,93 @@ export const stopAllConsumers = async (): Promise => { await stopUnifiedConsumer(channel); } }; + +/** + * Get list of currently running consumer channels + */ +export const getActiveConsumerChannels = (): string[] => { + return Array.from(consumers.keys()); +}; + +/** + * Check whether consumers for specified channels (or all active consumers) are healthy. + * + * @param expectedChannels Optional list of channels expected to be running. + * @param maxHeartbeatStalenessMs Max ms since last heartbeat before marking stalled (default 90s). + */ +export const areUnifiedConsumersHealthy = ( + expectedChannels?: string[], + maxHeartbeatStalenessMs = 90000 +): ConsumerHealthCheckResult => { + const details: Record = {}; + const now = Date.now(); + let allHealthy = true; + + const channelsToCheck = expectedChannels && expectedChannels.length > 0 + ? expectedChannels + : Array.from(consumers.keys()); + + // In standby mode (no channels expected/running), considered healthy + if (channelsToCheck.length === 0) { + return { healthy: true, details: {} }; + } + + for (const channel of channelsToCheck) { + const health = consumerHealthState.get(channel); + const consumer = consumers.get(channel); + + if (!health || !consumer) { + details[channel] = { + running: false, + crashed: false, + error: 'Consumer not initialized or missing', + secondsSinceHeartbeat: -1 + }; + allHealthy = false; + continue; + } + + const secondsSinceHeartbeat = Math.round((now - health.lastHeartbeat) / 1000); + + if (health.hasCrashed) { + details[channel] = { + running: false, + crashed: true, + error: health.crashReason || 'Consumer crashed', + secondsSinceHeartbeat + }; + allHealthy = false; + continue; + } + + if (!health.isRunning) { + details[channel] = { + running: false, + crashed: false, + error: 'Consumer is stopped or disconnected', + secondsSinceHeartbeat + }; + allHealthy = false; + continue; + } + + if (now - health.lastHeartbeat > maxHeartbeatStalenessMs) { + details[channel] = { + running: true, + crashed: false, + error: `Heartbeat stalled (${secondsSinceHeartbeat}s ago)`, + secondsSinceHeartbeat + }; + allHealthy = false; + continue; + } + + details[channel] = { + running: true, + crashed: false, + secondsSinceHeartbeat + }; + } + + return { healthy: allHealthy, details }; +}; diff --git a/src/processors/unified/unified.processor.ts b/src/processors/unified/unified.processor.ts index bae6baa..80f0197 100644 --- a/src/processors/unified/unified.processor.ts +++ b/src/processors/unified/unified.processor.ts @@ -1,204 +1,359 @@ /** * Unified Processor - Main entry point for plugin-based notification processing * - * Supports both single-channel and multi-channel modes: + * Supports dynamic MongoDB configuration, Redis Pub/Sub syncing, + * idle standby mode when empty, and both single-channel and multi-channel modes: * - Single: PROCESSOR_CHANNEL=email (or --channel email) * - Multi: PROCESSOR_CHANNEL=all (or omit to process all configured channels) */ +import mongoose from 'mongoose'; +import { connectMongoDB } from '@src/config/db.config.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { connectRedis, disconnectRedis } from '@src/config/redis.config.js'; -import { initStatusProducer, disconnectStatusProducer } from '@src/processors/shared/status.producer.js'; -import { initDelayedProducer, disconnectDelayedProducer } from '@src/processors/shared/delayed.producer.js'; -import { loadProvidersFromEnv, PluginRegistry } from '@src/plugins/index.js'; -import { startUnifiedConsumer, stopAllConsumers } from './unified.consumer.js'; +import { initStatusProducer, disconnectStatusProducer, isStatusProducerActive } from '@src/processors/shared/status.producer.js'; +import { initDelayedProducer, disconnectDelayedProducer, isDelayedProducerActive } from '@src/processors/shared/delayed.producer.js'; +import { + PluginRegistry, + PluginSyncService, + PluginManagerService, + NpmAuthService, + ProviderManagerService, + ChannelRoutingService, + loadProvidersFromDatabase, +} from '@src/plugins/index.js'; +import { + startUnifiedConsumer, + stopUnifiedConsumer, + stopAllConsumers, + getActiveConsumerChannels, + areUnifiedConsumersHealthy, +} from './unified.consumer.js'; import { unifiedProcessorLogger as logger } from './unified.logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; +import { createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeServer } from '@src/types/types.js'; +import { getRedisClient } from '@src/config/redis.config.js'; -//Import the admin channel provider files here for them to self-register +// Import the admin channel provider files here for them to self-register import "@src/admin-alerts/channels/discord.channel.js"; import "@src/admin-alerts/channels/telegram.channel.js"; // Track active channels for shutdown const activeChannels: string[] = []; let isShuttingDown = false; +let dbConnection: { disconnect: () => Promise } | null = null; +let probeServer: HealthProbeServer | null = null; /** * Parse channel argument from CLI or environment */ const getChannelConfig = (): string[] | 'all' => { - // Check CLI args first: --channel email - const channelArgIndex = process.argv.indexOf('--channel'); - if (channelArgIndex !== -1 && process.argv[channelArgIndex + 1]) { - const channel = process.argv[channelArgIndex + 1]; - return channel === 'all' ? 'all' : [channel]; - } + const channelArgIndex = process.argv.indexOf('--channel'); + if (channelArgIndex !== -1 && process.argv[channelArgIndex + 1]) { + const channel = process.argv[channelArgIndex + 1]; + return channel === 'all' ? 'all' : [channel]; + } + + const envChannel = process.env.PROCESSOR_CHANNEL; + if (envChannel) { + return envChannel === 'all' ? 'all' : [envChannel]; + } + + return 'all'; +}; - // Fall back to environment variable - const envChannel = process.env.PROCESSOR_CHANNEL; - if (envChannel) { - return envChannel === 'all' ? 'all' : [envChannel]; - } +/** + * Compute the list of channels this processor is currently configured to handle + */ +const getExpectedChannels = (): string[] => { + const loadedChannels = PluginRegistry.getChannels(); + const channelConfig = getChannelConfig(); + + if (channelConfig === 'all') { + return loadedChannels; + } - // Default: process all channels - return 'all'; + return channelConfig.filter((ch) => loadedChannels.includes(ch)); +}; + +/** + * Check for new channels and start consumers if configured + */ +const syncActiveConsumers = async (): Promise => { + if (isShuttingDown) return; + + const currentRunning = getActiveConsumerChannels(); + const expectedChannels = getExpectedChannels(); + + for (const channel of expectedChannels) { + if (!currentRunning.includes(channel)) { + try { + logger.info(`Starting dynamic consumer for new channel: ${channel}`); + await startUnifiedConsumer(channel); + if (!activeChannels.includes(channel)) { + activeChannels.push(channel); + } + } catch (err) { + logger.error(`Failed to dynamically start consumer for channel '${channel}':`, err); + } + } + } + + // If a channel was removed or is no longer expected, stop its consumer + for (const runningChannel of currentRunning) { + if (!expectedChannels.includes(runningChannel)) { + logger.info(`Stopping consumer for removed channel: ${runningChannel}`); + await stopUnifiedConsumer(runningChannel); + const index = activeChannels.indexOf(runningChannel); + if (index !== -1) { + activeChannels.splice(index, 1); + } + } + } }; /** * Graceful shutdown handler */ const gracefulShutdown = async (signal: string): Promise => { - if (isShuttingDown) { - logger.info('Shutdown already in progress...'); - return; + if (isShuttingDown) { + logger.info('Shutdown already in progress...'); + return; + } + + isShuttingDown = true; + logger.info(`Received ${signal}. Starting graceful shutdown...`); + + try { + // 1. Stop all consumers + logger.info('Stopping consumers...'); + await stopAllConsumers(); + + // 2. Shutdown plugins + logger.info('Shutting down plugins...'); + await PluginRegistry.shutdownAll(); + + // 3. Disconnect Kafka producers + logger.info('Disconnecting Kafka producers...'); + await disconnectStatusProducer(); + await disconnectDelayedProducer(); + + // 4. Disconnect Redis + logger.info('Disconnecting Redis...'); + await disconnectRedis(); + + // 5. Disconnect MongoDB + if (dbConnection) { + logger.info('Disconnecting MongoDB...'); + await dbConnection.disconnect(); } - isShuttingDown = true; - logger.info(`Received ${signal}. Starting graceful shutdown...`); - - try { - // 1. Stop all consumers - logger.info('Stopping consumers...'); - await stopAllConsumers(); - - // 2. Shutdown plugins - logger.info('Shutting down plugins...'); - await PluginRegistry.shutdownAll(); - - // 3. Disconnect Kafka producers - logger.info('Disconnecting Kafka producers...'); - await disconnectStatusProducer(); - await disconnectDelayedProducer(); - - // 4. Disconnect Redis - logger.info('Disconnecting Redis...'); - await disconnectRedis(); - - logger.success('Graceful shutdown complete'); - process.exit(0); - } catch (err) { - logger.error('Error during graceful shutdown:', err); - process.exit(1); + // 6. Stop health probe server + if (probeServer) { + await probeServer.stop(); } + + logger.success('Graceful shutdown complete'); + process.exit(0); + } catch (err) { + logger.error('Error during graceful shutdown:', err); + process.exit(1); + } }; /** * Register shutdown handlers */ const registerShutdownHandlers = (): void => { - process.on('SIGTERM', () => gracefulShutdown('SIGTERM')); - process.on('SIGINT', () => gracefulShutdown('SIGINT')); - - process.on('uncaughtException', async (err) => { - logger.error('Uncaught exception:', err); - - void AdminAlertService.sendAlert('service_health', - `🔴 UNCAUGHT EXCEPTION - Unified Processor\n` + - `Channels: ${activeChannels.join(', ')}\n` + - `Error: ${err instanceof Error ? err.message : String(err)}\n` + - `Stack: ${err instanceof Error ? err.stack?.slice(0, 200) : 'N/A'}\n` + - `Action: Check processor logs. Process is shutting down.`, - { severity: 'critical' }); - - await gracefulShutdown('uncaughtException'); - }); - - process.on('unhandledRejection', async (reason) => { - logger.error('Unhandled rejection:', reason); - - void AdminAlertService.sendAlert('service_health', - `🔴 UNHANDLED REJECTION - Unified Processor\n` + - `Channels: ${activeChannels.join(', ')}\n` + - `Reason: ${reason instanceof Error ? reason.message : String(reason)}\n` + - `Action: Check processor logs. Process is shutting down.`, - { severity: 'critical' }); - - await gracefulShutdown('unhandledRejection'); - }); + process.on('SIGTERM', () => gracefulShutdown('SIGTERM')); + process.on('SIGINT', () => gracefulShutdown('SIGINT')); + + process.on('uncaughtException', async (err) => { + logger.error('Uncaught exception:', err); + + void AdminAlertService.sendAlert( + 'service_health', + `🔴 UNCAUGHT EXCEPTION - Unified Processor\n` + + `Channels: ${activeChannels.join(', ')}\n` + + `Error: ${err instanceof Error ? err.message : String(err)}\n` + + `Stack: ${err instanceof Error ? err.stack?.slice(0, 200) : 'N/A'}\n` + + `Action: Check processor logs. Process is shutting down.`, + { severity: 'critical' } + ); + + await gracefulShutdown('uncaughtException'); + }); + + process.on('unhandledRejection', async (reason) => { + logger.error('Unhandled rejection:', reason); + + void AdminAlertService.sendAlert( + 'service_health', + `🔴 UNHANDLED REJECTION - Unified Processor\n` + + `Channels: ${activeChannels.join(', ')}\n` + + `Reason: ${reason instanceof Error ? reason.message : String(reason)}\n` + + `Action: Check processor logs. Process is shutting down.`, + { severity: 'critical' } + ); + + await gracefulShutdown('unhandledRejection'); + }); }; /** * Main entry point */ const main = async (): Promise => { - logger.info('================================'); - logger.info('Starting Unified Processor...'); - - try { - // 1. Connect to Redis - logger.info('Connecting to Redis...'); - await connectRedis(); - - // 2. Load plugins from configuration - logger.info('Loading plugins from configuration...'); - await loadProvidersFromEnv(); - - const loadedChannels = PluginRegistry.getChannels(); - if (loadedChannels.length === 0) { - logger.warn('No plugins loaded! Check simplens.config.yaml'); - } else { - logger.info(`Loaded plugins for channels: ${loadedChannels.join(', ')}`); - } + logger.info('================================'); + logger.info('Starting Unified Processor...'); + + try { + // 1. Connect to MongoDB + logger.info('Connecting to MongoDB...'); + dbConnection = await connectMongoDB(); + logger.success('Successfully connected to MongoDB'); + + // Initialize Dynamic Configuration + await dynamicConfig.initialize(false); + + // 2. Connect to Redis + logger.info('Connecting to Redis...'); + await connectRedis(); + + // 3. Initialize Redis Pub/Sub synchronization + await PluginSyncService.startSubscriber(); + NpmAuthService.registerSyncHandlers(); + await NpmAuthService.syncNpmrc(); + PluginManagerService.registerSyncHandlers(); + ProviderManagerService.registerSyncHandlers(); + ChannelRoutingService.registerSyncHandlers(); + + // Sync consumers when reload or channel route changes occur + PluginSyncService.on('SYSTEM_RELOAD', async () => { + logger.info('Received SYSTEM_RELOAD. Reloading providers from MongoDB...'); + await loadProvidersFromDatabase({ initialize: true }); + await syncActiveConsumers(); + }); - // 3. Initialize Kafka producers - logger.info('Initializing Kafka producers...'); - await initStatusProducer(); - await initDelayedProducer(); - - // 4. Determine which channels to process - const channelConfig = getChannelConfig(); - let channelsToProcess: string[]; - - if (channelConfig === 'all') { - channelsToProcess = loadedChannels; - logger.info('Running in multi-channel mode (all channels)'); - } else { - // Validate requested channels - channelsToProcess = channelConfig.filter(ch => { - if (!loadedChannels.includes(ch)) { - logger.warn(`Channel '${ch}' not found in loaded plugins, skipping`); - return false; - } - return true; - }); - logger.info(`Running in single-channel mode: ${channelsToProcess.join(', ')}`); - } + PluginSyncService.on('CHANNEL_ROUTING_UPDATED', async () => { + await syncActiveConsumers(); + }); - if (channelsToProcess.length === 0) { - logger.error('No valid channels to process! Exiting...'); + PluginSyncService.on('PROVIDER_UPSERTED', async () => { + await syncActiveConsumers(); + }); - void AdminAlertService.sendAlert('service_health', - `🔴 STARTUP FAILURE - Unified Processor\n` + - `Error: No valid channels to process\n` + - `Action: Check simplens.config.yaml. Verify plugins are installed.`, - { severity: 'critical' }); + PluginSyncService.on('PROVIDER_DELETED', async () => { + await syncActiveConsumers(); + }); - process.exit(1); - } + // 4. Load plugins and providers from MongoDB + logger.info('Loading providers from MongoDB...'); + await loadProvidersFromDatabase({ initialize: true }); - // 5. Start consumers for each channel - for (const channel of channelsToProcess) { - logger.info(`Starting consumer for channel: ${channel}`); - await startUnifiedConsumer(channel); - activeChannels.push(channel); - } + const loadedChannels = PluginRegistry.getChannels(); + if (loadedChannels.length === 0) { + logger.warn('No providers configured in MongoDB yet. Processor entering idle standby mode...'); + } else { + logger.info(`Loaded plugins for channels: ${loadedChannels.join(', ')}`); + } - // 6. Register shutdown handlers - registerShutdownHandlers(); + // 5. Initialize Kafka producers + logger.info('Initializing Kafka producers...'); + await initStatusProducer(); + await initDelayedProducer(); + + // 6. Determine which channels to process + const channelsToProcess = getExpectedChannels(); + if (getChannelConfig() === 'all') { + logger.info('Configured for multi-channel mode (all channels)'); + } else { + logger.info(`Configured for single-channel mode: ${channelsToProcess.join(', ')}`); + } - logger.info('================================'); - logger.success(`Unified Processor is running! Channels: ${activeChannels.join(', ')}`); + // 7. Start consumers for each channel (or wait in standby mode) + if (channelsToProcess.length === 0) { + logger.info('Unified Processor is in idle standby mode waiting for channel and provider configuration.'); + } else { + for (const channel of channelsToProcess) { + logger.info(`Starting consumer for channel: ${channel}`); + await startUnifiedConsumer(channel); + activeChannels.push(channel); + } + logger.success(`Unified Processor is running! Active channels: ${activeChannels.join(', ')}`); + } - } catch (err) { - logger.error('Failed to start Unified Processor:', err); + // 8. Start health probe server for Kubernetes + probeServer = createHealthProbeServer({ + serviceName: 'unified-processor', + readinessChecks: [ + { + name: 'mongodb', + check: () => !isShuttingDown && mongoose.connection.readyState === 1 + }, + { + name: 'redis', + check: () => { + if (isShuttingDown) return false; + try { + return getRedisClient().status === 'ready'; + } catch { + return false; + } + } + }, + { + name: 'kafka_producers', + check: () => !isShuttingDown && isStatusProducerActive() && isDelayedProducerActive() + }, + { + name: 'kafka_consumers', + check: () => { + if (isShuttingDown) return false; + const expected = getExpectedChannels(); + const result = areUnifiedConsumersHealthy(expected); + return result.healthy; + } + } + ], + livenessChecks: [ + { + name: 'process', + check: () => !isShuttingDown + }, + { + name: 'kafka_consumers', + check: () => { + if (isShuttingDown) return false; + const expected = getExpectedChannels(); + // Standby mode (no channels configured yet) is considered healthy + if (expected.length === 0) return true; + const result = areUnifiedConsumersHealthy(expected); + return result.healthy; + } + } + ] + }); + await probeServer.start(); - void AdminAlertService.sendAlert('service_health', - `🔴 STARTUP FAILURE - Unified Processor\n` + - `Error: ${err instanceof Error ? err.message : String(err)}\n` + - `Action: Check processor logs. Verify Redis and Kafka connectivity.`, - { severity: 'critical' }); + // 9. Register shutdown handlers + registerShutdownHandlers(); - process.exit(1); - } + logger.info('================================'); + } catch (err) { + logger.error('Failed to start Unified Processor:', err); + + void AdminAlertService.sendAlert( + 'service_health', + `🔴 STARTUP FAILURE - Unified Processor\n` + + `Error: ${err instanceof Error ? err.message : String(err)}\n` + + `Action: Check processor logs. Verify Redis, MongoDB, and Kafka connectivity.`, + { severity: 'critical' } + ); + + process.exit(1); + } }; // Start the processor diff --git a/src/types/schemas.ts b/src/types/schemas.ts index 099ca7b..5bb24c2 100644 --- a/src/types/schemas.ts +++ b/src/types/schemas.ts @@ -15,7 +15,6 @@ import { } from "./types.js"; import type { UUID } from "crypto"; import { validate, version } from "uuid"; -import { env } from "@src/config/env.config.js"; // ============================================================================ // BASE FIELD SCHEMAS @@ -41,26 +40,62 @@ const hasNonEmptyContent = ( content?: Record>, ) => !!content && Object.keys(content).length > 0; -// ============================================================================ -// BASE NOTIFICATION SCHEMA (Channel-Agnostic) -// ============================================================================ +export const providerAttemptSchema = z.object({ + provider: z.string(), + status: z.enum(['delivered', 'failed', 'rate_limited']), + error_code: z.string().optional(), + error_message: z.string().optional(), + retryable: z.boolean().optional(), + attempted_at: z.coerce.date(), + duration_ms: z.number().optional(), + response: z.unknown().optional(), +}); -/** - * Base notification schema - plugins extend this with channel-specific fields - */ -export const baseNotificationSchema = z.object({ - notification_id: objectIdSchema, +const notificationPayloadFields = { request_id: UUIDV4Schema, client_id: UUIDV4Schema, client_name: z.string().optional(), channel: z.string(), provider: z.string().optional(), + provider_history: z.array(providerAttemptSchema).optional(), recipient: z.record(z.string(), z.unknown()), content: z.record(z.string(), z.unknown()), variables: variablesSchema.optional(), webhook_url: z.url(), - retry_count: z.number().int().min(0), scheduled_at: z.coerce.date().optional(), +}; + +const notificationRequestFields = { + client_id: UUIDV4Schema, + client_name: z.string().optional(), + channel: z.array(z.string()).min(1, "At least one channel is required."), + template_id: z.array(z.string().nullable().optional()).optional(), + provider: z + .union([z.string(), z.array(z.string().nullable().optional())]) + .optional(), + content: z.record(z.string(), z.record(z.string(), z.string())).optional(), + scheduled_at: z.coerce.date().optional(), + webhook_url: z.url(), +}; + +const notificationTemplateFields = { + name: z.string(), + description: z.string().optional(), + content: z.record(z.string(), z.unknown()), + package: z.string(), +}; + +// ============================================================================ +// BASE NOTIFICATION SCHEMA (Channel-Agnostic) +// ============================================================================ + +/** + * Base notification schema - plugins extend this with channel-specific fields + */ +export const baseNotificationSchema = z.object({ + notification_id: objectIdSchema, + ...notificationPayloadFields, + retry_count: z.number().int().min(0), created_at: z.coerce.date(), }); @@ -89,6 +124,8 @@ export const notificationStatusTopicSchema = z.object({ request_id: UUIDV4Schema, client_id: UUIDV4Schema, channel: z.string(), + provider: z.string().optional(), + provider_history: z.array(providerAttemptSchema).optional(), status: z.enum(NOTIFICATION_STATUS_SF), message: z.string(), retry_count: z.number().int().min(0), @@ -104,19 +141,11 @@ export const notificationStatusTopicSchema = z.object({ * Notification record in MongoDB */ export const notificationSchema = z.object({ - request_id: UUIDV4Schema, - client_id: UUIDV4Schema, - client_name: z.string().optional(), - channel: z.string(), - provider: z.string().optional(), - recipient: z.record(z.string(), z.unknown()), - content: z.record(z.string(), z.unknown()), - variables: variablesSchema.optional(), - webhook_url: z.url(), + ...notificationPayloadFields, status: z.enum(NOTIFICATION_STATUS), - scheduled_at: z.coerce.date().optional(), error_message: z.string().optional(), retry_count: z.number().int().min(0), + api_key_id: z.string().nullable().optional(), // Recovery claiming fields for horizontal scalability recovery_claimed_by: z.string().nullable().optional(), recovery_claimed_at: z.coerce.date().nullable().optional(), @@ -164,6 +193,8 @@ export const statusOutboxSchema = z.object({ _id: objectIdSchema, notification_id: objectIdSchema, status: z.enum(NOTIFICATION_STATUS_SF), + provider: z.string().optional(), + provider_history: z.array(providerAttemptSchema).optional(), processed: z.boolean().default(false), claimed_by: z.string().nullable().optional(), claimed_at: z.coerce.date().nullable().optional(), @@ -243,7 +274,91 @@ export const adminChannelSchema = z.object({ */ export const systemConfigSchema = z.object({ key: z.string(), - value: z.string(), + value: z.unknown(), + created_at: z.coerce.date().optional(), + updated_at: z.coerce.date().optional(), +}); + +/** + * Key-pair envelope encrypted credentials schema + */ +export const encryptedCredentialsSchema = z.object({ + encrypted_data: z.string(), + iv: z.string(), + auth_tag: z.string(), + encrypted_dek: z.string(), +}); + +/** + * Provider manifest schema for plugins + */ +export const providerManifestSchema = z.object({ + name: z.string(), + version: z.string(), + channel: z.string(), + displayName: z.string(), + description: z.string(), + author: z.string().optional(), + homepage: z.string().optional(), + requiredCredentials: z.array(z.string()), + optionalConfig: z.array(z.string()).optional(), +}); + + /** Options supplied by a YAML/JSON provider configuration entry. */ + export const providerConfigOptionsSchema = z.object({ + priority: z.number().int().optional(), + rateLimit: z.object({ + maxTokens: z.number().nonnegative().optional(), + refillRate: z.number().nonnegative().optional(), + refillInterval: z.enum(['second', 'minute', 'hour', 'day']).optional(), + }).optional(), + }).catchall(z.unknown()); + + /** Provider entry as read from YAML/JSON configuration. */ + export const providerConfigEntrySchema = z.object({ + package: z.string().min(1), + version: z.string().optional(), + id: z.string().min(1), + credentials: z.record(z.string(), z.string()).optional(), + optionalConfig: z.record(z.string(), z.string()).optional(), + options: providerConfigOptionsSchema.optional(), + }); + +/** + * Plugin document schema + */ +export const pluginSchema = z.object({ + name: z.string().min(1), + version: z.string().min(1), + status: z.enum(['installed', 'installing', 'failed']), + error: z.string().optional(), + manifest: providerManifestSchema, + created_at: z.coerce.date().optional(), + updated_at: z.coerce.date().optional(), +}); + +/** + * Provider document schema + */ +export const providerSchema = z.object({ + id: z.string().min(1), + plugin_name: z.string().min(1), + channel: z.string().min(1), + enabled: z.boolean().default(true), + credentials: encryptedCredentialsSchema, + options: providerConfigOptionsSchema.optional(), + created_at: z.coerce.date().optional(), + updated_at: z.coerce.date().optional(), +}); + +/** + * Channel routing document schema + */ +export const channelRoutingSchema = z.object({ + channel: z.string().min(1), + default_provider_id: z.string().min(1), + fallback_provider_ids: z.array(z.string()).default([]), + partitions: z.number().int().min(1).default(6), created_at: z.coerce.date().optional(), updated_at: z.coerce.date().optional(), }); @@ -276,18 +391,9 @@ export const telegramConfigSchema = z.object({ export const baseNotificationRequestSchema = z .object({ request_id: UUIDV4Schema, - client_id: UUIDV4Schema, - client_name: z.string().optional(), - template_id: z.array(z.string().nullable().optional()).optional(), - channel: z.array(z.string()).min(1, "At least one channel is required."), - provider: z - .union([z.string(), z.array(z.string().nullable().optional())]) - .optional(), + ...notificationRequestFields, recipient: z.record(z.string(), z.unknown()), - content: z.record(z.string(), z.record(z.string(), z.string())).optional(), variables: variablesSchema.optional(), - scheduled_at: z.coerce.date().optional(), - webhook_url: z.url(), }) .superRefine((data, ctx) => { if ( @@ -310,14 +416,7 @@ export const baseNotificationRequestSchema = z */ export const baseBatchNotificationRequestSchema = z .object({ - client_id: UUIDV4Schema, - client_name: z.string().optional(), - channel: z.array(z.string()).min(1, "At least one channel is required."), - template_id: z.array(z.string().nullable().optional()).optional(), - provider: z - .union([z.string(), z.array(z.string().nullable().optional())]) - .optional(), - content: z.record(z.string(), z.record(z.string(), z.string())).optional(), + ...notificationRequestFields, recipients: z.array( z.looseObject({ request_id: UUIDV4Schema, @@ -325,25 +424,23 @@ export const baseBatchNotificationRequestSchema = z variables: variablesSchema.optional(), }), ).min(1, "At least one recipient is required."), - scheduled_at: z.coerce.date().optional(), - webhook_url: z.url(), }) - .refine( - (data) => { - // limit check - if ( - data.recipients && - data.recipients.length * data.channel.length > env.MAX_BATCH_REQ_LIMIT - ) { - return false; - } - return true; - }, - { - message: `Batch size exceeds limit (${env.MAX_BATCH_REQ_LIMIT})`, - path: ["recipients"], - }, - ) + .superRefine((data, ctx) => { + const maxLimit = + typeof process !== "undefined" && process.env.MAX_BATCH_REQ_LIMIT + ? parseInt(process.env.MAX_BATCH_REQ_LIMIT) + : 1000; + if ( + data.recipients && + data.recipients.length * data.channel.length > maxLimit + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Batch size exceeds limit (${maxLimit})`, + path: ["recipients"], + }); + } + }) .superRefine((data, ctx) => { if ( !hasNonEmptyTemplateIds(data.template_id) && @@ -364,19 +461,11 @@ export const baseBatchNotificationRequestSchema = z Notification Template Request Schema */ export const notificationTemplateRequestSchema = z.object({ - name: z.string(), + ...notificationTemplateFields, template_id: z.string().optional(), - description: z.string().optional(), - content: z.record(z.string(), z.unknown()), - package: z.string(), }); -export const notificationTemplateUpdateRequestSchema = z.object({ - name: z.string(), - description: z.string().optional(), - content: z.record(z.string(), z.unknown()), - package: z.string(), -}); +export const notificationTemplateUpdateRequestSchema = z.object(notificationTemplateFields); // ============================================================================ // VALIDATION FUNCTIONS @@ -422,3 +511,262 @@ export const safeValidateNotificationTemplateRequestSchema = (data: unknown) => notificationTemplateRequestSchema.safeParse(data); export const safeValidateNotificationTemplateUpdateRequestSchema = (data: unknown) => notificationTemplateUpdateRequestSchema.safeParse(data); + +// ============================================================================ +// DYNAMIC OPERATIONAL SETTINGS & ADMIN AUTH SCHEMAS +// ============================================================================ + +export const operationalSettingsSchema = z.object({ + api: z.object({ + max_batch_req_limit: z.number().int().min(10).max(10000).default(1000), + }), + worker: z.object({ + outbox_poll_interval_ms: z.number().int().min(500).max(60000).default(5000), + outbox_cleanup_interval_ms: z.number().int().min(5000).max(600000).default(60000), + outbox_batch_size: z.number().int().min(1).max(2000).default(100), + outbox_retention_ms: z.number().int().min(10000).max(86400000).default(300000), + outbox_claim_timeout_ms: z.number().int().min(5000).max(300000).default(30000), + }), + retry: z.object({ + max_retry_count: z.number().int().min(1).max(30).default(5), + idempotency_ttl_seconds: z.number().int().min(60).max(2592000).default(86400), + processing_ttl_seconds: z.number().int().min(10).max(1800).default(120), + rate_limit_retry_delay_ms: z.number().int().min(100).max(60000).default(5000), + }), + delayed: z.object({ + delayed_poll_interval_ms: z.number().int().min(200).max(30000).default(1000), + delayed_batch_size: z.number().int().min(1).max(500).default(10), + max_poller_retries: z.number().int().min(1).max(10).default(3), + }), + recovery: z.object({ + recovery_poll_interval_ms: z.number().int().min(5000).max(600000).default(60000), + processing_stuck_threshold_ms: z.number().int().min(10000).max(3600000).default(300000), + pending_stuck_threshold_ms: z.number().int().min(10000).max(3600000).default(300000), + recovery_batch_size: z.number().int().min(1).max(500).default(50), + recovery_claim_timeout_ms: z.number().int().min(5000).max(300000).default(60000), + cleanup_resolved_alerts_retention_ms: z.number().int().min(60000).max(2592000000).default(86400000), + cleanup_processed_status_outbox_retention_ms: z.number().int().min(60000).max(2592000000).default(86400000), + }), + logging: z.object({ + log_level: z.enum(['debug', 'info', 'warn', 'error']).default('info'), + log_to_file: z.boolean().default(true), + }), +}); + +export const partialOperationalSettingsSchema = z.object({ + api: operationalSettingsSchema.shape.api.partial().optional(), + worker: operationalSettingsSchema.shape.worker.partial().optional(), + retry: operationalSettingsSchema.shape.retry.partial().optional(), + delayed: operationalSettingsSchema.shape.delayed.partial().optional(), + recovery: operationalSettingsSchema.shape.recovery.partial().optional(), + logging: operationalSettingsSchema.shape.logging.partial().optional(), +}); + +export const adminSetupSchema = z.object({ + username: z.string().trim().min(3, "Username must be at least 3 characters").max(50, "Username cannot exceed 50 characters"), + password: z.string().min(8, "Password must be at least 8 characters").max(64, "Password cannot exceed 64 characters"), +}); + +export const adminLoginSchema = z.object({ + username: z.string().trim().min(1, "Username is required"), + password: z.string().min(1, "Password is required"), +}); + +/** + * npm registry configuration schema for individual custom registries/scopes + */ +export const npmRegistryConfigSchema = z.object({ + id: z.string().optional(), + scope: z.string().trim().optional(), + registry_url: z.string().url("Invalid registry URL").default("https://registry.npmjs.org/"), + token: z.string().min(1, "npm token cannot be empty"), +}); + +/** + * npm registry status item for public/dashboard display + */ +export const npmRegistryStatusSchema = z.object({ + id: z.string(), + scope: z.string().optional(), + registry_url: z.string(), + masked_token: z.string(), + updated_at: z.string(), +}); + +/** + * npm registry authentication config schema + */ +export const npmAuthConfigSchema = z.object({ + token: z.string().min(1, "npm token cannot be empty"), + registry_url: z.string().url("Invalid registry URL").default("https://registry.npmjs.org/"), + scope: z.string().trim().optional(), +}); + +/** + * npm registry auth status schema (safe to return to UI with masked token and registry list) + */ +export const npmAuthStatusSchema = z.object({ + is_configured: z.boolean(), + registries: z.array(npmRegistryStatusSchema).default([]), + default_registry: z.string().optional(), + masked_token: z.string().optional(), +}); + +/** + * Payload schema for installing a plugin package + */ +/** + * Strict npm package name schema preventing shell injection and metacharacters + */ +export const npmPackageNameSchema = z.string() + .min(1, 'Package name is required') + .max(214, 'Package name cannot exceed 214 characters') + .regex(/^(?:@[a-z0-9_.-]+\/)?[a-z0-9_.-]+$/i, 'Invalid npm package name format'); + +/** + * Strict npm package version / tag schema + */ +export const npmPackageVersionSchema = z.string() + .regex( + /^(latest|\^?[0-9]+(?:\.[0-9]+)?(?:\.[0-9]+)?(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?|~?[0-9]+(?:\.[0-9]+)?(?:\.[0-9]+)?(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?|[0-9]+(?:\.[0-9]+)?(?:\.[0-9]+)?(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)$/, + 'Invalid npm version or tag format' + ); + +/** + * Payload schema for changing plugin version + */ +export const changePluginVersionSchema = z.object({ + package: npmPackageNameSchema, + version: npmPackageVersionSchema, +}); + +/** + * Schema for uninstalling a plugin + */ +export const uninstallPluginSchema = z.object({ + package: npmPackageNameSchema, +}); + +export const installPluginPayloadSchema = z.object({ + package: npmPackageNameSchema, + version: npmPackageVersionSchema.optional(), + auth: z.object({ + token: z.string().optional(), + registry_url: z.string().url("Invalid registry URL").optional(), + scope: z.string().optional(), + save_token: z.boolean().default(true), + }).optional(), +}); + +/** + * Provider rate limit configuration schema + */ +export const providerRateLimitConfigDetailsSchema = z.object({ + max_tokens: z.number(), + refill_rate: z.number(), + refill_interval: z.enum(['second', 'minute', 'hour', 'day']), + normalized_refill_rate: z.number(), +}); + +/** + * Provider rate limit real-time status schema + */ +export const providerRateLimitRealtimeStatusSchema = z.object({ + remaining_tokens: z.number(), + used_tokens: z.number(), + usage_percentage: z.number(), + is_exhausted: z.boolean(), + resets_in_ms: z.number(), + full_refill_in_ms: z.number(), + last_refill_at: z.coerce.date().nullable(), + last_exhausted_at: z.coerce.date().nullable(), + exhausted_count: z.number(), +}); + +/** + * Provider rate limit status item schema + */ +export const providerRateLimitStatusSchema = z.object({ + provider_id: z.string(), + channel: z.string(), + plugin_name: z.string(), + display_name: z.string().optional(), + enabled: z.boolean(), + config: providerRateLimitConfigDetailsSchema, + status: providerRateLimitRealtimeStatusSchema, +}); + +/** + * Provider rate limit summary schema + */ +export const providerRateLimitSummarySchema = z.object({ + total_providers: z.number(), + exhausted_providers: z.number(), + healthy_providers: z.number(), +}); + +/** + * Provider rate limit list response schema + */ +export const providerRateLimitListResponseSchema = z.object({ + providers: z.array(providerRateLimitStatusSchema), + summary: providerRateLimitSummarySchema, +}); + +// ============================================================================ +// API KEY & ADMIN AUTH SCHEMAS +// ============================================================================ + +export const apiKeyUsageSchema = z.object({ + total_requests: z.number().int().min(0).default(0), + total_notifications: z.number().int().min(0).default(0), + by_channel: z.record(z.string(), z.number().int().min(0)).default({}), + last_used_at: z.coerce.date().nullable().optional(), +}); + +export const apiKeyStatusSchema = z.enum(['active', 'revoked']); + +export const apiKeyTypeSchema = z.enum(['notification', 'admin']); + +export const apiKeyDocSchema = z.object({ + key_id: z.string(), + name: z.string(), + type: apiKeyTypeSchema.default('notification'), + key_prefix: z.string(), + key_hash: z.string(), + status: apiKeyStatusSchema, + expires_at: z.coerce.date().nullable().optional(), + usage: apiKeyUsageSchema, + created_at: z.coerce.date().optional(), + updated_at: z.coerce.date().optional(), +}); + +export const createApiKeySchema = z.object({ + name: z.string().min(1, 'API key name is required').max(100), + type: apiKeyTypeSchema.default('notification'), + expires_at: z.coerce.date().optional(), +}); + +export const apiKeyResponseSchema = z.object({ + key_id: z.string(), + name: z.string(), + type: apiKeyTypeSchema, + key_prefix: z.string(), + status: apiKeyStatusSchema, + expires_at: z.coerce.date().nullable().optional(), + usage: apiKeyUsageSchema, + created_at: z.coerce.date(), +}); + +export const createApiKeyResponseSchema = z.object({ + key: apiKeyResponseSchema, + raw_key: z.string(), +}); + +export const apiHealthResponseSchema = z.object({ + service: z.string().optional(), + status: z.enum(['healthy', 'unhealthy', 'ready', 'not_ready']), + timestamp: z.string(), + checks: z.record(z.string(), z.boolean()), +}); + diff --git a/src/types/types.ts b/src/types/types.ts index a3de578..a3eaa27 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -63,6 +63,41 @@ export enum ALERT_TYPE { orphaned_pending = "orphaned_pending" } +export type RefillInterval = 'second' | 'minute' | 'hour' | 'day'; + +export interface DeliveryResult { + success: boolean; + messageId?: string; + provider?: string; + providerResponse?: unknown; + error?: { + code: string; + message: string; + retryable: boolean; + }; + attempts?: ProviderAttempt[]; +} + +export interface ProviderConfig { + id: string; + credentials: Record; + options?: ProviderOptions; +} + +export interface SimpleNSProvider { + readonly manifest: ProviderManifest; + + getNotificationSchema(): z.ZodSchema; + getRecipientSchema(): z.ZodObject>; + getContentSchema(): z.ZodObject>; + getRateLimitConfig(): RateLimitConfig; + + initialize(config: ProviderConfig): Promise; + healthCheck(): Promise; + send(notification: TNotification): Promise; + shutdown(): Promise; +} + // ============================================================================ // SCHEMA IMPORTS // ============================================================================ @@ -77,6 +112,7 @@ import { baseNotificationRequestSchema, baseBatchNotificationRequestSchema, baseNotificationSchema, + providerAttemptSchema, // Admin channel schemas adminChannelSchema, systemConfigSchema, @@ -87,6 +123,39 @@ import { ADMIN_CHANNEL_TYPE, ADMIN_ALERT_TYPE, notificationTemplateSchema, + // Plugin & Provider schemas + encryptedCredentialsSchema, + providerManifestSchema, + providerConfigOptionsSchema, + providerConfigEntrySchema, + pluginSchema, + providerSchema, + channelRoutingSchema, + operationalSettingsSchema, + partialOperationalSettingsSchema, + adminSetupSchema, + adminLoginSchema, + npmRegistryConfigSchema, + npmRegistryStatusSchema, + npmAuthConfigSchema, + npmAuthStatusSchema, + installPluginPayloadSchema, + changePluginVersionSchema, + uninstallPluginSchema, + npmPackageNameSchema, + npmPackageVersionSchema, + providerRateLimitConfigDetailsSchema, + providerRateLimitRealtimeStatusSchema, + providerRateLimitStatusSchema, + providerRateLimitSummarySchema, + providerRateLimitListResponseSchema, + apiKeyUsageSchema, + apiKeyStatusSchema, + apiKeyTypeSchema, + apiKeyDocSchema, + createApiKeySchema, + apiKeyResponseSchema, + createApiKeyResponseSchema, } from "./schemas.js"; // ============================================================================ @@ -111,6 +180,27 @@ export type status_outbox = z.infer; export type base_notification = z.infer; +export type ProviderAttempt = z.infer; + +export type BaseNotification = Omit & { + notification_id: string; +}; +export type ProviderManifest = provider_manifest; +export type RateLimitConfig = { + maxTokens: number; + refillRate: number; + refillInterval?: RefillInterval; +}; + +export interface RateLimitResult { + allowed: boolean; + remainingTokens: number; + retryAfterMs?: number; + queuePosition?: number; +} + +export type HealthChecker = () => Promise; + // ============================================================================ // ADMIN NOTIFICATION CHANNEL TYPES // ============================================================================ @@ -124,3 +214,443 @@ export type telegram_config = z.infer; export type AdminChannelType = (typeof ADMIN_CHANNEL_TYPE)[number]; export type AdminAlertType = (typeof ADMIN_ALERT_TYPE)[number]; export type notification_template = z.infer; + +// ============================================================================ +// PLUGIN & PROVIDER MANAGEMENT TYPES +// ============================================================================ + +export type encrypted_credentials = z.infer; +export type provider_manifest = z.infer; +export type plugin_document = z.infer; +export type provider_document = z.infer; +export type channel_routing_document = z.infer; +export type npm_registry_config = z.infer; +export type npm_registry_status = z.infer; +export type npm_auth_config = z.infer; +export type npm_auth_status = z.infer; +export type install_plugin_payload = z.infer; + +export interface KafkaConsumerState { + consumer: import('kafkajs').Consumer | null; + isConsuming: boolean; +} + +export interface OutboxCronState { + pollIntervalId: NodeJS.Timeout | null; + cleanupIntervalId: NodeJS.Timeout | null; + statusPollIntervalId: NodeJS.Timeout | null; + isPolling: boolean; + isCleaningUp: boolean; + isPollingStatus: boolean; + shouldStop: boolean; +} + +export interface RecoveryCronState { + intervalId: NodeJS.Timeout | null; + isRunning: boolean; + shouldStop: boolean; + healthChecker: HealthChecker | null; + consecutiveFailures: number; +} + +export interface ChannelResult { + success: boolean; + error?: string; +} + +export interface AlertMetadata { + alertType?: AdminAlertType; + severity?: 'info' | 'warning' | 'critical'; + timestamp?: Date; + notificationId?: string; + channel?: string; + errorMessage?: string; +} + +export interface CredentialField { + name: string; + type: 'string' | 'url' | 'secret'; + label: string; + placeholder?: string; + description?: string; + required: boolean; + pattern?: string; +} + +export interface AdminChannelProvider { + readonly channelType: AdminChannelType; + readonly displayName: string; + send(message: string, metadata?: AlertMetadata): Promise; + testConnection(): Promise; + getCredentialSchema(): CredentialField[]; +} + +export interface AdminChannelMeta { + channelType: AdminChannelType; + displayName: string; + credentialFields: CredentialField[]; +} + +export interface RegisteredProvider { + provider: SimpleNSProvider; + id: string; + priority: number; +} + +export interface ChannelConfig { + default: string; + fallback?: string | string[]; +} + +export interface FieldDefinition { + name: string; + type: 'string' | 'email' | 'phone' | 'text' | 'number' | 'boolean'; + required: boolean; + description?: string; +} + +export interface ProviderMetadata { + id: string; + name: string; + displayName: string; + description?: string; + priority: number; + recipientFields: FieldDefinition[]; + contentFields: FieldDefinition[]; +} + +export interface ChannelMetadata { + providers: ProviderMetadata[]; + default?: string; + fallback?: string | string[]; +} + +export interface PluginMetadata { + channels: Record; +} + +export type ProviderRateLimitOptions = z.infer['rateLimit']; +export type ProviderOptions = z.infer; + +export type ProviderConfigEntry = z.infer; + +export type plugin_catalog_entry = { + name: string; + package: string; + description: string; + versions?: string[]; +}; +export type YamlProviderEntry = ProviderConfigEntry; + +export interface YamlConfig { + providers?: YamlProviderEntry[]; + channels?: Record; +} + +export type ProviderEntry = ProviderConfigEntry & { + credentials: Record; +}; + +export interface SimpleNSConfig { + providers: ProviderEntry[]; + channels?: Record; +} + +export interface ProviderResponseDto { + _id?: string; + id: string; + plugin_name: string; + channel: string; + enabled: boolean; + options?: ProviderOptions; + credentials_configured: boolean; + created_at?: Date; + updated_at?: Date; +} + +export type PluginSyncAction = + | 'PLUGIN_INSTALLED' + | 'PLUGIN_UNINSTALLED' + | 'PLUGIN_VERSION_CHANGED' + | 'PROVIDER_UPSERTED' + | 'PROVIDER_DELETED' + | 'CHANNEL_ROUTING_UPDATED' + | 'SYSTEM_RELOAD' + | 'NPM_AUTH_UPDATED'; + +export interface PluginSyncPayload { + plugin_name?: string; + version?: string; + provider_id?: string; + channel?: string; + [key: string]: unknown; +} + +export interface PluginSyncMessage { + event_id: string; + source_instance_id: string; + timestamp: string; + action: PluginSyncAction; + payload: PluginSyncPayload; +} + +export type PluginSyncHandler = (message: PluginSyncMessage) => Promise; + +export interface DelayedEventWithRetries extends delayed_notification_topic { + _pollerRetries?: number; +} + +export interface IdempotencyRecord { + status: 'processing' | 'delivered' | 'failed' | 'rate_limited'; + retry_count: number; + updated_at: string; +} + +export interface StatusProcessResult { + dbUpdated: boolean; + webhookUrl?: string; + webhookPayload?: WebhookPayload; + notificationId?: string; +} + +export interface WebhookPayload { + request_id: string; + client_id: string; + notification_id: string; + status: string; + channel: string; + message: string; + occurred_at: string; +} + +export type ServiceContext = + | 'api' + | 'producer' + | 'consumer' + | 'cron' + | 'worker' + | 'delayedWorker' + | 'unifiedProcessor' + | 'redis' + | 'recoveryService' + | 'adminAlert' + | 'pluginLoader' + | 'pluginRegistry' + | 'rateLimiter' + | 'configSync'; + +export interface LogMeta { + notificationId?: string; + requestId?: string; + clientId?: string; + channel?: string; + workerId?: string; + topic?: string; + partition?: number; + [key: string]: unknown; +} + +export interface Logger { + info: (message: string, meta?: LogMeta) => void; + warn: (message: string, meta?: LogMeta) => void; + error: (message: string, meta?: LogMeta | unknown) => void; + debug: (message: string, meta?: LogMeta) => void; + success: (message: string, meta?: LogMeta) => void; +} + +export interface SendResult { + successCount: number; + failedCount: number; +} + +export interface ProcessingStats { + processed: number; + success: number; + failed: number; +} + +export interface ValidatedOutboxEntry { + _id: import('mongoose').Types.ObjectId; + notification_id: import('mongoose').Types.ObjectId | string; + topic: string; + payload: Record; + status: string; +} + +// ============================================================================ +// DYNAMIC OPERATIONAL SETTINGS & ADMIN AUTH TYPES +// ============================================================================ + +export type OperationalSettings = z.infer; +export type PartialOperationalSettings = z.infer; +export type OperationalSettingsGroup = keyof OperationalSettings; + +export type SystemConfigSyncAction = 'SETTINGS_SEEDED' | 'SETTINGS_UPDATED' | 'SETTINGS_RESET'; + +export interface SystemConfigSyncMessage { + action: SystemConfigSyncAction; + source_instance_id: string; + timestamp: string; + payload?: PartialOperationalSettings; +} + +export interface AdminCredentialsDoc { + username: string; + password_hash: string; + salt: string; + created_at: string; + updated_at?: string; +} + +export type AdminSetupPayload = z.infer; +export type AdminLoginPayload = z.infer; + +export interface AdminAuthStatus { + isConfigured: boolean; +} + +// ============================================================================ +// KUBERNETES HEALTH PROBE TYPES +// ============================================================================ + +export interface HealthProbeCheck { + name: string; + check: () => Promise | boolean; +} + +export interface HealthProbeOptions { + port?: number; + serviceName: string; + readinessChecks?: HealthProbeCheck[]; + livenessChecks?: HealthProbeCheck[]; +} + +export interface HealthProbeServer { + start: () => Promise; + stop: () => Promise; +} + +export interface HealthCheckResultDetails { + ok: boolean; + results: Record; +} + +export interface ApiHealthResponse { + service?: string; + status: 'healthy' | 'unhealthy' | 'ready' | 'not_ready'; + timestamp: string; + checks: Record; +} + +// ============================================================================ +// PROCESSOR CONSUMER HEALTH TYPES +// ============================================================================ + +export interface ConsumerHealthState { + channel: string; + isRunning: boolean; + hasCrashed: boolean; + crashReason?: string; + lastHeartbeat: number; +} + +export interface ConsumerHealthDetails { + running: boolean; + crashed: boolean; + error?: string; + secondsSinceHeartbeat: number; +} + +export interface ConsumerHealthCheckResult { + healthy: boolean; + details: Record; +} + +// ============================================================================ +// PROVIDER RATE LIMIT STATUS TYPES +// ============================================================================ + +export type ProviderRateLimitConfigDetails = z.infer; +export type ProviderRateLimitRealtimeStatus = z.infer; +export type ProviderRateLimitStatus = z.infer; +export type ProviderRateLimitSummary = z.infer; +export type ProviderRateLimitListResponse = z.infer; + +export interface ProviderBasicMeta { + id: string; + channel: string; + plugin_name: string; + display_name?: string; + enabled: boolean; +} + +export type npm_package_name = z.infer; +export type npm_package_version = z.infer; +export type change_plugin_version_payload = z.infer; +export type uninstall_plugin_payload = z.infer; + +// ============================================================================ +// API KEY & ADMIN AUTH TYPES +// ============================================================================ + +export interface AdminJwtPayload { + sub: string; + username: string; + role: 'admin'; + iat?: number; + exp?: number; +} + +export type ApiKeyType = z.infer; +export type ApiKeyStatus = z.infer; +export type ApiKeyUsage = z.infer; +export type ApiKeyDoc = z.infer; +export type CreateApiKeyInput = z.infer; +export type ApiKeyResponse = z.infer; +export type CreateApiKeyResponse = z.infer; + +export interface ApiKeyCacheEntry { + keyDoc: ApiKeyDoc; + cachedAt: number; +} + +export interface PendingApiKeyUsage { + total_requests: number; + total_notifications: number; + by_channel: Record; + last_used_at: Date; +} + +// ============================================================================ +// NPM AUTH TYPES +// ============================================================================ + +export interface StoredRegistryEntry { + id: string; + scope?: string; + registry_url: string; + encrypted_token: encrypted_config; + updated_at: string; +} + +export interface StoredNpmAuthConfig { + registries?: StoredRegistryEntry[]; + // Legacy fields for backwards compatibility + encrypted_token?: encrypted_config; + registry_url?: string; +} + +export interface DecryptedRegistryEntry { + id: string; + scope?: string; + registry_url: string; + token: string; + updated_at: string; +} \ No newline at end of file diff --git a/src/utils/backoff.utils.ts b/src/utils/backoff.utils.ts new file mode 100644 index 0000000..89e2322 --- /dev/null +++ b/src/utils/backoff.utils.ts @@ -0,0 +1,22 @@ +/** + * Backoff calculation utilities + */ + +/** + * Calculate exponential backoff delay in milliseconds + * Formula: Math.min(baseDelayMs * Math.pow(2, retryCount), maxDelayMs) + * + * @param retryCount - Current retry attempt count (>= 0) + * @param baseDelayMs - Base delay in milliseconds (default: 1000ms) + * @param maxDelayMs - Maximum delay cap in milliseconds (default: 300000ms / 5 minutes) + * @returns Delay in milliseconds + */ +export function calculateExponentialBackoff( + retryCount: number, + baseDelayMs = 1000, + maxDelayMs = 300000 +): number { + const safeRetryCount = Math.max(0, retryCount); + const delay = baseDelayMs * Math.pow(2, safeRetryCount); + return Math.min(delay, maxDelayMs); +} diff --git a/src/utils/k8s-health-probe.ts b/src/utils/k8s-health-probe.ts new file mode 100644 index 0000000..9395d56 --- /dev/null +++ b/src/utils/k8s-health-probe.ts @@ -0,0 +1,115 @@ +/** + * Lightweight Kubernetes Health Probe Server + * + * Provides HTTP endpoints (/healthz, /readyz) for background workers + * and processors to integrate cleanly with Kubernetes liveness and readiness probes. + * + * Uses Node's built-in http module for zero runtime overhead. + */ + +import http from 'http'; +import type { HealthCheckResultDetails, HealthProbeCheck, HealthProbeOptions, HealthProbeServer } from '@src/types/types.js'; + +export const runHealthChecks = async ( + checks: HealthProbeCheck[] = [] +): Promise => { + const results: Record = {}; + let allOk = true; + + for (const item of checks) { + try { + const passed = await item.check(); + results[item.name] = Boolean(passed); + if (!passed) allOk = false; + } catch { + results[item.name] = false; + allOk = false; + } + } + + return { ok: allOk, results }; +}; + +export const createHealthProbeServer = (options: HealthProbeOptions): HealthProbeServer => { + const port = options.port ?? parseInt(process.env.PROBE_PORT || '8080', 10); + const serviceName = options.serviceName; + const readinessChecks = options.readinessChecks ?? []; + const livenessChecks = options.livenessChecks ?? []; + + let server: http.Server | null = null; + + const requestListener: http.RequestListener = async (req, res) => { + const url = req.url || ''; + + // Liveness probe + if (url === '/healthz' || url === '/livez') { + const { ok, results } = await runHealthChecks(livenessChecks); + const status = ok ? 200 : 503; + res.writeHead(status, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + service: serviceName, + status: ok ? 'healthy' : 'unhealthy', + timestamp: new Date().toISOString(), + checks: results + })); + return; + } + + // Readiness probe + if (url === '/readyz' || url === '/ready') { + const { ok, results } = await runHealthChecks(readinessChecks); + const status = ok ? 200 : 503; + res.writeHead(status, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + service: serviceName, + status: ok ? 'ready' : 'not_ready', + timestamp: new Date().toISOString(), + checks: results + })); + return; + } + + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Not Found' })); + }; + + return { + start: async (): Promise => { + // Allow disabling health probe explicitly via environment variable + if (process.env.ENABLE_HEALTH_PROBE === 'false') { + return; + } + + return new Promise((resolve, reject) => { + server = http.createServer(requestListener); + + server.on('error', (err: NodeJS.ErrnoException) => { + // If port is already in use (e.g. running multiple services locally in one container), + // log and do not crash the service + if (err.code === 'EADDRINUSE') { + console.warn(`[HealthProbe:${serviceName}] Port ${port} is already in use, health probe disabled.`); + resolve(); + return; + } + reject(err); + }); + + server.listen(port, () => { + console.info(`[HealthProbe:${serviceName}] Listening on port ${port} (/healthz, /readyz)`); + resolve(); + }); + }); + }, + + stop: async (): Promise => { + if (!server) return; + + return new Promise((resolve) => { + server!.close(() => { + server = null; + resolve(); + }); + }); + } + }; +}; diff --git a/src/workers/consumers/status.consumer.ts b/src/workers/consumers/status.consumer.ts index 9b30f84..10a2a1d 100644 --- a/src/workers/consumers/status.consumer.ts +++ b/src/workers/consumers/status.consumer.ts @@ -1,6 +1,6 @@ -import { Consumer, EachMessagePayload } from "kafkajs"; +import { EachMessagePayload } from "kafkajs"; import { kafka } from "@src/config/kafka.config.js"; -import { CORE_TOPICS, NOTIFICATION_STATUS, NOTIFICATION_STATUS_SF, type notification_status_topic } from "@src/types/types.js"; +import { CORE_TOPICS, NOTIFICATION_STATUS, NOTIFICATION_STATUS_SF, type KafkaConsumerState, type notification_status_topic, type StatusProcessResult, type WebhookPayload } from "@src/types/types.js"; import { safeValidateNotificationStatusTopic } from "@src/types/schemas.js"; import notification_model from "@src/database/models/notification.models.js"; import { consumerLogger as logger } from "@src/workers/utils/logger.js"; @@ -11,16 +11,15 @@ const CONSUMER_GROUP_ID = "notification-status-group"; const WEBHOOK_TIMEOUT_MS = 30000; // 30 seconds // Consumer state management -interface ConsumerState { - consumer: Consumer | null; - isConsuming: boolean; -} - -const state: ConsumerState = { +const state: KafkaConsumerState = { consumer: null, isConsuming: false }; +let lastHeartbeat = Date.now(); +let hasCrashed = false; +let crashReason: string | undefined = undefined; + /** * Map status from external format to internal notification status */ @@ -33,7 +32,7 @@ const mapToNotificationStatus = (externalStatus: NOTIFICATION_STATUS_SF): NOTIFI /** * Build webhook payload from status data */ -const buildWebhookPayload = (data: notification_status_topic) => ({ +const buildWebhookPayload = (data: notification_status_topic): WebhookPayload => ({ request_id: data.request_id, client_id: data.client_id, notification_id: data.notification_id.toString(), @@ -89,18 +88,11 @@ const sendWebhookCallback = ( /** * Result of processing a status message */ -interface ProcessResult { - dbUpdated: boolean; - webhookUrl?: string; - webhookPayload?: ReturnType; - notificationId?: string; -} - /** * Process a single status message * Returns result indicating if DB was updated and webhook info */ -const processStatusMessage = async ({ partition, message }: EachMessagePayload): Promise => { +const processStatusMessage = async ({ partition, message }: EachMessagePayload): Promise => { try { if (!message.value) { logger.warn("Received empty message, skipping"); @@ -125,6 +117,14 @@ const processStatusMessage = async ({ partition, message }: EachMessagePayload): updated_at: new Date() }; + if (data.provider) { + updateData.provider = data.provider; + } + + if (data.provider_history && data.provider_history.length > 0) { + updateData.provider_history = data.provider_history; + } + // Store error message if failed if (newStatus === NOTIFICATION_STATUS.failed) { updateData.error_message = data.message; @@ -133,7 +133,7 @@ const processStatusMessage = async ({ partition, message }: EachMessagePayload): const result = await notification_model.findByIdAndUpdate( data.notification_id, updateData, - { new: true } + { returnDocument: 'after' } ); if (result) { @@ -176,6 +176,40 @@ export const startStatusConsumer = async (): Promise => { await state.consumer.connect(); logger.info("Status consumer connected"); + lastHeartbeat = Date.now(); + hasCrashed = false; + crashReason = undefined; + + if (typeof state.consumer.on === 'function' && state.consumer.events) { + state.consumer.on(state.consumer.events.HEARTBEAT, () => { + lastHeartbeat = Date.now(); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + state.consumer.on(state.consumer.events.CRASH, (event: any) => { + hasCrashed = true; + state.isConsuming = false; + const crashError = event?.payload?.error; + crashReason = crashError ? String(crashError.message || crashError) : 'Unknown crash'; + logger.error(`Status consumer crashed: ${crashReason}`); + }); + + state.consumer.on(state.consumer.events.STOP, () => { + state.isConsuming = false; + logger.warn('Status consumer stopped event received'); + }); + + state.consumer.on(state.consumer.events.DISCONNECT, () => { + state.isConsuming = false; + logger.warn('Status consumer disconnected'); + }); + + state.consumer.on(state.consumer.events.CONNECT, () => { + state.isConsuming = true; + lastHeartbeat = Date.now(); + }); + } + await state.consumer.subscribe({ topic: CORE_TOPICS.notification_status, fromBeginning: false @@ -187,6 +221,7 @@ export const startStatusConsumer = async (): Promise => { await state.consumer.run({ autoCommit: false, eachMessage: async (payload) => { + lastHeartbeat = Date.now(); try { const result = await processStatusMessage(payload); @@ -228,6 +263,7 @@ export const stopStatusConsumer = async (): Promise => { logger.info("Stopping status consumer..."); state.isConsuming = false; + hasCrashed = false; try { await state.consumer.stop(); @@ -246,3 +282,14 @@ export const stopStatusConsumer = async (): Promise => { export const isStatusConsumerRunning = (): boolean => { return state.consumer !== null && state.isConsuming; }; + +/** + * Check if the status consumer is healthy (running, not crashed, heartbeat fresh) + */ +export const isStatusConsumerHealthy = (maxHeartbeatStalenessMs = 90000): boolean => { + if (!state.consumer || !state.isConsuming || hasCrashed) { + return false; + } + const isStale = (Date.now() - lastHeartbeat) > maxHeartbeatStalenessMs; + return !isStale; +}; diff --git a/src/workers/cron/background.cron.ts b/src/workers/cron/background.cron.ts index 878187a..2e10fa2 100644 --- a/src/workers/cron/background.cron.ts +++ b/src/workers/cron/background.cron.ts @@ -1,5 +1,6 @@ import { env } from "@src/config/env.config.js"; -import { OUTBOX_STATUS } from "@src/types/types.js"; +import { dynamicConfig } from "@src/config/dynamic-config.service.js"; +import { OUTBOX_STATUS, type OutboxCronState } from "@src/types/types.js"; import outbox_model from "@src/database/models/outbox.models.js"; import status_outbox_model from "@src/database/models/status-outbox.models.js"; import { type status_outbox } from "@src/types/types.js"; @@ -8,17 +9,7 @@ import { cronLogger as logger } from "@src/workers/utils/logger.js"; import type { OutboxDocument } from "@src/workers/utils/validation.js"; // Cron state management -interface CronState { - pollIntervalId: NodeJS.Timeout | null; - cleanupIntervalId: NodeJS.Timeout | null; - statusPollIntervalId: NodeJS.Timeout | null; - isPolling: boolean; - isCleaningUp: boolean; - isPollingStatus: boolean; - shouldStop: boolean; -} - -const state: CronState = { +const state: OutboxCronState = { pollIntervalId: null, cleanupIntervalId: null, statusPollIntervalId: null, @@ -30,28 +21,32 @@ const state: CronState = { /** * Atomically claim a batch of pending outbox entries for this worker. - * Uses findOneAndUpdate in a loop to ensure no two workers claim the same event. - * Also reclaims stale entries from crashed workers. + * Uses a 2-phase batch claim to eliminate sequential round trips and lock contention: + * 1. Fast-path: query up to OUTBOX_BATCH_SIZE pending entries in FIFO order using index { status: 1, created_at: 1 } + * 2. Atomically update matching pending IDs to 'processing' with claimed_by and claimed_at + * 3. Fallback: if fewer than batch size claimed, reclaim stale entries from crashed workers */ const claimOutboxEntries = async (): Promise => { const claimedEntries: OutboxDocument[] = []; const now = new Date(); - const staleThreshold = new Date(now.getTime() - env.OUTBOX_CLAIM_TIMEOUT_MS); + const batchSize = env.OUTBOX_BATCH_SIZE; - // Claim entries one at a time atomically (or use bulkWrite for better performance) - for (let i = 0; i < env.OUTBOX_BATCH_SIZE; i++) { - // Try to claim a pending entry, or reclaim a stale processing entry - const entry = await outbox_model.findOneAndUpdate( + // 1. Fast-path: Find pending entries in FIFO order + const pendingCandidates = await outbox_model + .find({ status: OUTBOX_STATUS.pending }) + .sort({ created_at: 1 }) + .limit(batchSize) + .select('_id') + .lean(); + + if (pendingCandidates.length > 0) { + const candidateIds = pendingCandidates.map(c => c._id); + + // Atomically claim entries that are still pending + await outbox_model.updateMany( { - $or: [ - // Unclaimed pending entries - { status: OUTBOX_STATUS.pending }, - // Stale entries from crashed workers (claimed but not processed in time) - { - status: OUTBOX_STATUS.processing, - claimed_at: { $lt: staleThreshold } - } - ] + _id: { $in: candidateIds }, + status: OUTBOX_STATUS.pending }, { $set: { @@ -59,22 +54,69 @@ const claimOutboxEntries = async (): Promise => { claimed_by: env.WORKER_ID, claimed_at: now } - }, - { - new: true, - sort: { created_at: 1 } // FIFO order } ); - if (!entry) break; // No more entries to claim - claimedEntries.push(entry); + // Retrieve entries successfully claimed by this worker + const claimed = await outbox_model.find({ + _id: { $in: candidateIds }, + claimed_by: env.WORKER_ID, + claimed_at: now + }); + + claimedEntries.push(...claimed); + } + + // 2. Fallback: If quota remaining, reclaim stale entries from crashed workers + const remainingQuota = batchSize - claimedEntries.length; + if (remainingQuota > 0) { + const staleThreshold = new Date(now.getTime() - env.OUTBOX_CLAIM_TIMEOUT_MS); + + const staleCandidates = await outbox_model + .find({ + status: OUTBOX_STATUS.processing, + claimed_at: { $lt: staleThreshold } + }) + .sort({ created_at: 1 }) + .limit(remainingQuota) + .select('_id') + .lean(); + + if (staleCandidates.length > 0) { + const staleIds = staleCandidates.map(c => c._id); + + await outbox_model.updateMany( + { + _id: { $in: staleIds }, + status: OUTBOX_STATUS.processing, + claimed_at: { $lt: staleThreshold } + }, + { + $set: { + status: OUTBOX_STATUS.processing, + claimed_by: env.WORKER_ID, + claimed_at: now + } + } + ); + + const reclaimed = await outbox_model.find({ + _id: { $in: staleIds }, + claimed_by: env.WORKER_ID, + claimed_at: now + }); + + claimedEntries.push(...reclaimed); + } } return claimedEntries; }; /** - * Poll the outbox collection for pending events and send them to Kafka + * Poll the outbox collection for pending events and send them to Kafka. + * Continuously drains backlog while full batches are retrieved, preventing + * idle intervals during high ingestion load. */ const pollOutbox = async (): Promise => { if (state.isPolling || state.shouldStop) return; @@ -82,15 +124,22 @@ const pollOutbox = async (): Promise => { state.isPolling = true; try { - // Atomically claim entries for this worker - const claimedEntries = await claimOutboxEntries(); + while (!state.shouldStop) { + // Atomically claim entries for this worker + const claimedEntries = await claimOutboxEntries(); - if (claimedEntries.length === 0) return; + if (claimedEntries.length === 0) break; - logger.info(`Claimed ${claimedEntries.length} outbox entries (worker: ${env.WORKER_ID})`); + logger.info(`Claimed ${claimedEntries.length} outbox entries (worker: ${env.WORKER_ID})`); - const result = await sendOutboxEvents(claimedEntries); - logger.info(`Processed: ${result.successCount} success, ${result.failedCount} failed`); + const result = await sendOutboxEvents(claimedEntries); + logger.info(`Processed: ${result.successCount} success, ${result.failedCount} failed`); + + // If we claimed fewer than batch size, queue is drained for now + if (claimedEntries.length < env.OUTBOX_BATCH_SIZE) { + break; + } + } } catch (err) { logger.error("Error polling outbox:", err); } finally { @@ -135,41 +184,95 @@ const waitForOperationsToComplete = async (): Promise => { /** * Atomically claim a batch of pending status outbox entries for this worker. + * Uses 2-phase atomic batch claim to avoid sequential findOneAndUpdate round trips. */ const claimStatusOutboxEntries = async (): Promise => { const claimedEntries: status_outbox[] = []; const now = new Date(); - const staleThreshold = new Date(now.getTime() - env.OUTBOX_CLAIM_TIMEOUT_MS); + const batchSize = env.OUTBOX_BATCH_SIZE; - for (let i = 0; i < env.OUTBOX_BATCH_SIZE; i++) { - const entry = await status_outbox_model.findOneAndUpdate( + // 1. Fast-path: Find unprocessed status outbox entries + const pendingCandidates = await status_outbox_model + .find({ processed: false, claimed_by: null }) + .sort({ created_at: 1 }) + .limit(batchSize) + .select('_id') + .lean(); + + if (pendingCandidates.length > 0) { + const candidateIds = pendingCandidates.map(c => c._id); + + await status_outbox_model.updateMany( { - $or: [ - { processed: false, claimed_by: null }, - { processed: false, claimed_at: { $lt: staleThreshold } } - ] + _id: { $in: candidateIds }, + processed: false, + claimed_by: null }, { $set: { claimed_by: env.WORKER_ID, claimed_at: now } - }, - { - new: true, - sort: { created_at: 1 } } ); - if (!entry) break; - claimedEntries.push(entry); + const claimed = await status_outbox_model.find({ + _id: { $in: candidateIds }, + claimed_by: env.WORKER_ID, + claimed_at: now + }); + + claimedEntries.push(...claimed); + } + + // 2. Fallback: Check for stale processing entries + const remainingQuota = batchSize - claimedEntries.length; + if (remainingQuota > 0) { + const staleThreshold = new Date(now.getTime() - env.OUTBOX_CLAIM_TIMEOUT_MS); + + const staleCandidates = await status_outbox_model + .find({ + processed: false, + claimed_at: { $lt: staleThreshold } + }) + .sort({ created_at: 1 }) + .limit(remainingQuota) + .select('_id') + .lean(); + + if (staleCandidates.length > 0) { + const staleIds = staleCandidates.map(c => c._id); + + await status_outbox_model.updateMany( + { + _id: { $in: staleIds }, + processed: false, + claimed_at: { $lt: staleThreshold } + }, + { + $set: { + claimed_by: env.WORKER_ID, + claimed_at: now + } + } + ); + + const reclaimed = await status_outbox_model.find({ + _id: { $in: staleIds }, + claimed_by: env.WORKER_ID, + claimed_at: now + }); + + claimedEntries.push(...reclaimed); + } } return claimedEntries; }; /** - * Poll and process status outbox entries + * Poll and process status outbox entries. + * Continuously drains backlog while full batches are retrieved. */ const pollStatusOutbox = async (): Promise => { if (state.isPollingStatus || state.shouldStop) return; @@ -177,15 +280,21 @@ const pollStatusOutbox = async (): Promise => { state.isPollingStatus = true; try { - const claimedEntries = await claimStatusOutboxEntries(); - if (claimedEntries.length === 0) { - return; - } + while (!state.shouldStop) { + const claimedEntries = await claimStatusOutboxEntries(); + if (claimedEntries.length === 0) { + break; + } + + logger.info(`Claimed ${claimedEntries.length} status outbox entries (worker: ${env.WORKER_ID})`); - logger.info(`Claimed ${claimedEntries.length} status outbox entries (worker: ${env.WORKER_ID})`); + const result = await sendStatusOutboxEvents(claimedEntries); + logger.info(`Status outbox: ${result.successCount} success, ${result.failedCount} failed`); - const result = await sendStatusOutboxEvents(claimedEntries); - logger.info(`Status outbox: ${result.successCount} success, ${result.failedCount} failed`); + if (claimedEntries.length < env.OUTBOX_BATCH_SIZE) { + break; + } + } } catch (err) { logger.error("Error polling status outbox:", err); } finally { @@ -220,6 +329,29 @@ export const startCronJobs = (): void => { logger.success("Cron jobs started"); }; +// Listen for dynamic configuration updates and hot-adjust intervals +dynamicConfig.on('change', () => { + if (state.shouldStop) return; + + if (state.pollIntervalId) { + clearInterval(state.pollIntervalId); + state.pollIntervalId = setInterval(pollOutbox, env.OUTBOX_POLL_INTERVAL_MS); + logger.info(`Adjusted outbox poll interval to ${env.OUTBOX_POLL_INTERVAL_MS}ms`); + } + + if (state.statusPollIntervalId) { + clearInterval(state.statusPollIntervalId); + state.statusPollIntervalId = setInterval(pollStatusOutbox, env.OUTBOX_POLL_INTERVAL_MS); + logger.info(`Adjusted status outbox poll interval to ${env.OUTBOX_POLL_INTERVAL_MS}ms`); + } + + if (state.cleanupIntervalId) { + clearInterval(state.cleanupIntervalId); + state.cleanupIntervalId = setInterval(cleanupPublishedEvents, env.OUTBOX_CLEANUP_INTERVAL_MS); + logger.info(`Adjusted outbox cleanup interval to ${env.OUTBOX_CLEANUP_INTERVAL_MS}ms`); + } +}); + /** * Stop the cron jobs gracefully */ diff --git a/src/workers/producers/background.producer.ts b/src/workers/producers/background.producer.ts index 5bbdf5f..5d18f3d 100644 --- a/src/workers/producers/background.producer.ts +++ b/src/workers/producers/background.producer.ts @@ -182,6 +182,8 @@ export const sendStatusOutboxEvents = async (statusEntries: status_outbox[]): Pr request_id: notification.request_id, client_id: notification.client_id, channel: notification.channel, + provider: entry.provider || notification.provider, + provider_history: entry.provider_history || notification.provider_history, status: entry.status as NOTIFICATION_STATUS_SF, message: entry.status === 'delivered' ? 'Recovered by recovery service - ghost delivery' @@ -216,3 +218,10 @@ export const sendStatusOutboxEvents = async (statusEntries: status_outbox[]): Pr return { successCount, failedCount }; }; + +/** + * Check if the Kafka producer is initialized and active + */ +export const isProducerActive = (): boolean => { + return producer !== null; +}; diff --git a/src/workers/recovery/recovery.cron.ts b/src/workers/recovery/recovery.cron.ts index 4d9754e..547865d 100644 --- a/src/workers/recovery/recovery.cron.ts +++ b/src/workers/recovery/recovery.cron.ts @@ -13,7 +13,8 @@ import mongoose from 'mongoose'; import { env } from '@src/config/env.config.js'; -import { NOTIFICATION_STATUS, ALERT_TYPE } from '@src/types/types.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; +import { NOTIFICATION_STATUS, ALERT_TYPE, type HealthChecker, type RecoveryCronState } from '@src/types/types.js'; import notification_model from '@src/database/models/notification.models.js'; import alert_model from '@src/database/models/alert.models.js'; import status_outbox_model from '@src/database/models/status-outbox.models.js'; @@ -22,21 +23,10 @@ import { recoveryLogger as logger } from '@src/workers/utils/logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; import { NOTIFICATION_STATUS_SF } from '@src/types/types.js'; -// Health checker function type -type HealthChecker = () => Promise; - -// Cron state management -interface CronState { - intervalId: NodeJS.Timeout | null; - isRunning: boolean; - shouldStop: boolean; - healthChecker: HealthChecker | null; - consecutiveFailures: number; -} - const MAX_CONSECUTIVE_FAILURES = 5; -const state: CronState = { +// Cron state management +const state: RecoveryCronState = { intervalId: null, isRunning: false, shouldStop: false, @@ -72,7 +62,7 @@ const recoverStuckProcessing = async (): Promise => { recovery_claimed_at: now } }, - { new: true } + { returnDocument: 'after' } ); if (!notification) break; @@ -302,7 +292,7 @@ const detectOrphanedPending = async (): Promise => { recovery_claimed_at: now } }, - { new: true } + { returnDocument: 'after' } ); if (!notification) break; @@ -414,7 +404,7 @@ const autoResolveDeliveredAlerts = async (): Promise => { recovery_claimed_at: now } }, - { new: true } + { returnDocument: 'after' } ); if (!alert) break; @@ -537,6 +527,16 @@ export const startRecoveryCron = (): void => { logger.success('Recovery cron started'); }; +// Listen for dynamic configuration updates and hot-adjust recovery interval +dynamicConfig.on('change', () => { + if (state.shouldStop) return; + if (state.intervalId) { + clearInterval(state.intervalId); + state.intervalId = setInterval(runRecovery, env.RECOVERY_POLL_INTERVAL_MS); + logger.info(`Adjusted recovery cron interval to ${env.RECOVERY_POLL_INTERVAL_MS}ms`); + } +}); + /** * Stop the recovery cron job gracefully */ diff --git a/src/workers/recovery/recovery.service.ts b/src/workers/recovery/recovery.service.ts index 6d8a8b4..f2b6502 100644 --- a/src/workers/recovery/recovery.service.ts +++ b/src/workers/recovery/recovery.service.ts @@ -14,10 +14,13 @@ import mongoose from 'mongoose'; import { env } from '@src/config/env.config.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; import { connectRedis, disconnectRedis, getRedisClient } from '@src/config/redis.config.js'; -import { startRecoveryCron, stopRecoveryCron, setHealthChecker } from './recovery.cron.js'; +import { startRecoveryCron, stopRecoveryCron, setHealthChecker, isRecoveryCronRunning } from './recovery.cron.js'; import { recoveryLogger as logger, flushLogs } from '@src/workers/utils/logger.js'; import { AdminAlertService } from '@src/admin-alerts/admin-alert.service.js'; +import { createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeServer } from '@src/types/types.js'; //Import the admin channel provider files here for them to self-register import "@src/admin-alerts/channels/discord.channel.js"; @@ -26,6 +29,7 @@ import "@src/admin-alerts/channels/telegram.channel.js"; let isShuttingDown = false; let mongoReconnecting = false; let redisReconnecting = false; +let probeServer: HealthProbeServer | null = null; const RECONNECT_DELAY_MS = 5000; const MAX_RECONNECT_ATTEMPTS = 10; @@ -239,6 +243,11 @@ const shutdown = async (signal: string): Promise => { // Flush logs before exit await flushLogs(); + // Stop health probe server + if (probeServer) { + await probeServer.stop(); + } + logger.success('Recovery service shutdown complete'); process.exit(0); } catch (err) { @@ -264,6 +273,10 @@ const main = async (): Promise => { logger.info('Recovery service will start anyway and retry connections...'); } + if (mongoConnected) { + await dynamicConfig.initialize(false); + } + // Set health checker for recovery cron setHealthChecker(checkHealth); @@ -275,6 +288,21 @@ const main = async (): Promise => { logger.info(`Processing stuck threshold: ${env.PROCESSING_STUCK_THRESHOLD_MS}ms`); logger.info(`Pending stuck threshold: ${env.PENDING_STUCK_THRESHOLD_MS}ms`); + // Start health probe server for Kubernetes + probeServer = createHealthProbeServer({ + serviceName: 'recovery-service', + readinessChecks: [ + { name: 'mongodb', check: () => !isShuttingDown && isMongoHealthy() }, + { name: 'redis', check: () => !isShuttingDown && isRedisHealthy() }, + { name: 'recovery_cron', check: () => !isShuttingDown && isRecoveryCronRunning() } + ], + livenessChecks: [ + { name: 'process', check: () => !isShuttingDown }, + { name: 'recovery_cron', check: () => !isShuttingDown && isRecoveryCronRunning() } + ] + }); + await probeServer.start(); + // Register signal handlers for graceful shutdown process.on('SIGTERM', () => shutdown('SIGTERM')); process.on('SIGINT', () => shutdown('SIGINT')); diff --git a/src/workers/utils/logger.ts b/src/workers/utils/logger.ts index f753d7e..429a9d9 100644 --- a/src/workers/utils/logger.ts +++ b/src/workers/utils/logger.ts @@ -5,7 +5,7 @@ import winston from 'winston'; import LokiTransport from 'winston-loki'; -import { env } from '@src/config/env.config.js'; +import type { LogMeta, Logger, ServiceContext } from '@src/types/types.js'; // Service context types const SERVICE_LABELS = { @@ -21,23 +21,10 @@ const SERVICE_LABELS = { adminAlert: 'admin-alert', pluginLoader: 'plugin-loader', pluginRegistry: 'plugin-registry', - rateLimiter: 'rate-limiter' + rateLimiter: 'rate-limiter', + configSync: 'config-sync' } as const; -type ServiceContext = keyof typeof SERVICE_LABELS; - -// Log metadata interface -interface LogMeta { - notificationId?: string; - requestId?: string; - clientId?: string; - channel?: string; - workerId?: string; - topic?: string; - partition?: number; - [key: string]: unknown; -} - // Custom log levels including 'success' const customLevels = { levels: { @@ -81,7 +68,8 @@ const SERVICE_EMOJI: Record = { adminAlert: '🔔', pluginLoader: '🔌', pluginRegistry: '📦', - rateLimiter: '⏱️' + rateLimiter: '⏱️', + configSync: '⚙️' }; /** @@ -124,15 +112,15 @@ const createWinstonLogger = (service: ServiceContext) => { ]; // Add Loki transport if LOKI_URL is configured - if (env.LOKI_URL) { + if (process.env.LOKI_URL) { transports.push( new LokiTransport({ - host: env.LOKI_URL, + host: process.env.LOKI_URL, labels: { job: 'notification-service', service: SERVICE_LABELS[service], - workerId: env.WORKER_ID, - environment: env.NODE_ENV + workerId: process.env.WORKER_ID || 'worker-default', + environment: process.env.NODE_ENV || 'development' }, json: true, batching: true, @@ -146,7 +134,9 @@ const createWinstonLogger = (service: ServiceContext) => { } // Add file transport in production - if (env.NODE_ENV === 'production' && env.LOG_TO_FILE) { + const isProd = (process.env.NODE_ENV || 'development') === 'production'; + const logToFile = process.env.LOG_TO_FILE === 'true'; + if (isProd && logToFile) { transports.push( new winston.transports.File({ filename: `logs/${SERVICE_LABELS[service]}.log`, @@ -164,27 +154,30 @@ const createWinstonLogger = (service: ServiceContext) => { ); } - return winston.createLogger({ + const winstonLogger = winston.createLogger({ levels: customLevels.levels, - level: env.LOG_LEVEL || 'info', + level: process.env.LOG_LEVEL || 'info', defaultMeta: { service: SERVICE_LABELS[service], - workerId: "worker-default" + workerId: process.env.WORKER_ID || "worker-default" }, transports }); + + activeWinstonLoggers.push(winstonLogger); + return winstonLogger; }; +const activeWinstonLoggers: winston.Logger[] = []; + /** - * Logger interface matching our usage patterns + * Dynamically update log level across all active loggers */ -export interface Logger { - info: (message: string, meta?: LogMeta) => void; - warn: (message: string, meta?: LogMeta) => void; - error: (message: string, meta?: LogMeta | unknown) => void; - debug: (message: string, meta?: LogMeta) => void; - success: (message: string, meta?: LogMeta) => void; -} +export const setLogLevel = (newLevel: string): void => { + for (const l of activeWinstonLoggers) { + l.level = newLevel; + } +}; /** * Create a typed logger for a service context @@ -231,6 +224,7 @@ export const pluginLoaderLogger = createLogger('pluginLoader'); export const pluginRegistryLogger = createLogger('pluginRegistry'); export const rateLimiterLogger = createLogger('rateLimiter'); export const adminAlertLogger = createLogger('adminAlert'); +export const configSyncLogger = createLogger('configSync'); /** * Graceful shutdown - flush all logs before exit diff --git a/src/workers/utils/types.ts b/src/workers/utils/types.ts index 2304527..9602a07 100644 --- a/src/workers/utils/types.ts +++ b/src/workers/utils/types.ts @@ -1,14 +1 @@ -/** - * Common types used across worker modules - */ - -export interface SendResult { - successCount: number; - failedCount: number; -} - -export interface ProcessingStats { - processed: number; - success: number; - failed: number; -} +export type { ProcessingStats, SendResult } from '@src/types/types.js'; diff --git a/src/workers/utils/validation.ts b/src/workers/utils/validation.ts index 4315795..c6b6eaf 100644 --- a/src/workers/utils/validation.ts +++ b/src/workers/utils/validation.ts @@ -5,7 +5,7 @@ */ import mongoose from "mongoose"; -import { type outbox } from "@src/types/types.js"; +import { type outbox, type ValidatedOutboxEntry } from "@src/types/types.js"; import { safeValidateOutbox, safeValidateBaseNotification, @@ -14,6 +14,8 @@ import { import type { HydratedDocument } from "mongoose"; import { producerLogger as logger } from "./logger.js"; +export type { ValidatedOutboxEntry } from '@src/types/types.js'; + // Type for outbox document from MongoDB export type OutboxDocument = HydratedDocument; @@ -21,14 +23,6 @@ export type OutboxDocument = HydratedDocument; export type ValidPayload = Record; // Validated outbox entry -export interface ValidatedOutboxEntry { - _id: mongoose.Types.ObjectId; - notification_id: mongoose.Types.ObjectId | string; - topic: string; - payload: ValidPayload; - status: string; -} - /** * Validate an outbox entry and its payload * @returns ValidatedOutboxEntry if valid, null if validation fails diff --git a/src/workers/worker.ts b/src/workers/worker.ts index 1a4fa8e..58c0fd3 100644 --- a/src/workers/worker.ts +++ b/src/workers/worker.ts @@ -1,9 +1,13 @@ +import mongoose from "mongoose"; import { connectMongoDB } from "@src/config/db.config.js"; -import { initProducer, disconnectProducer } from "@src/workers/producers/background.producer.js"; -import { startCronJobs, stopCronJobs } from "@src/workers/cron/background.cron.js"; -import { startStatusConsumer, stopStatusConsumer } from "@src/workers/consumers/status.consumer.js"; +import { dynamicConfig } from "@src/config/dynamic-config.service.js"; +import { initProducer, disconnectProducer, isProducerActive } from "@src/workers/producers/background.producer.js"; +import { startCronJobs, stopCronJobs, isCronRunning } from "@src/workers/cron/background.cron.js"; +import { startStatusConsumer, stopStatusConsumer, isStatusConsumerHealthy } from "@src/workers/consumers/status.consumer.js"; import { workerLogger as logger } from "@src/workers/utils/logger.js"; import { AdminAlertService } from "@src/admin-alerts/admin-alert.service.js"; +import { createHealthProbeServer } from "@src/utils/k8s-health-probe.js"; +import type { HealthProbeServer } from "@src/types/types.js"; //Import the admin channel provider files here for them to self-register import "@src/admin-alerts/channels/discord.channel.js"; @@ -11,6 +15,7 @@ import "@src/admin-alerts/channels/telegram.channel.js"; let isShuttingDown = false; let dbConnection: Awaited> | null = null; +let probeServer: HealthProbeServer | null = null; /** * Graceful shutdown handler @@ -43,6 +48,11 @@ const gracefulShutdown = async (signal: string): Promise => { await dbConnection.disconnect(); } + // 5. Stop health probe server + if (probeServer) { + await probeServer.stop(); + } + logger.success("Graceful shutdown complete"); process.exit(0); } catch (err) { @@ -97,6 +107,9 @@ const main = async (): Promise => { dbConnection = await connectMongoDB(); logger.success("MongoDB connected"); + // Initialize Dynamic Configuration + await dynamicConfig.initialize(false); + // 2. Initialize Kafka producer logger.info("Initializing Kafka producer..."); await initProducer(); @@ -113,6 +126,44 @@ const main = async (): Promise => { logger.success("Background Worker is running!"); logger.info("================================"); + // 5. Start health probe server for Kubernetes + probeServer = createHealthProbeServer({ + serviceName: 'worker', + readinessChecks: [ + { + name: 'mongodb', + check: () => !isShuttingDown && mongoose.connection.readyState === 1 + }, + { + name: 'status_consumer', + check: () => !isShuttingDown && isStatusConsumerHealthy() + }, + { + name: 'cron_jobs', + check: () => !isShuttingDown && isCronRunning() + }, + { + name: 'kafka_producer', + check: () => !isShuttingDown && isProducerActive() + } + ], + livenessChecks: [ + { + name: 'process', + check: () => !isShuttingDown + }, + { + name: 'status_consumer', + check: () => !isShuttingDown && isStatusConsumerHealthy() + }, + { + name: 'cron_jobs', + check: () => !isShuttingDown && isCronRunning() + } + ] + }); + await probeServer.start(); + // Register shutdown handlers registerShutdownHandlers(); } catch (err) { diff --git a/tests/integration/api/admin-auth.test.ts b/tests/integration/api/admin-auth.test.ts new file mode 100644 index 0000000..569638c --- /dev/null +++ b/tests/integration/api/admin-auth.test.ts @@ -0,0 +1,150 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import admin_auth_router from '@src/api/routes/admin-auth.routes.js'; +import system_config_model from '@src/database/models/system-config.models.js'; + +vi.mock('@src/database/models/system-config.models.js', () => ({ + default: { + findOne: vi.fn(), + create: vi.fn(), + findOneAndUpdate: vi.fn(), + }, +})); + +describe('Admin Auth Routes Integration', () => { + let app: express.Application; + + beforeEach(() => { + vi.clearAllMocks(); + app = express(); + app.use(express.json()); + app.use('/api/admin/auth', admin_auth_router); + }); + + describe('GET /api/admin/auth/status', () => { + it('should return isConfigured: false when no credentials exist in database', async () => { + vi.mocked(system_config_model.findOne).mockResolvedValue(null); + + const res = await request(app).get('/api/admin/auth/status'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ isConfigured: false }); + }); + + it('should return isConfigured: true when credentials exist in database', async () => { + vi.mocked(system_config_model.findOne).mockResolvedValue({ + key: 'admin_credentials', + value: { username: 'admin' }, + } as any); + + const res = await request(app).get('/api/admin/auth/status'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ isConfigured: true }); + }); + }); + + describe('POST /api/admin/auth/signup', () => { + it('should reject setup with validation error if password too short', async () => { + const res = await request(app) + .post('/api/admin/auth/signup') + .send({ username: 'admin', password: '123' }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Validation failed'); + }); + + it('should reject setup with validation error if password exceeds max length', async () => { + const res = await request(app) + .post('/api/admin/auth/signup') + .send({ username: 'admin', password: 'a'.repeat(65) }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Validation failed'); + }); + + it('should reject setup if already configured', async () => { + vi.mocked(system_config_model.findOne).mockResolvedValue({ + key: 'admin_credentials', + value: { username: 'admin' }, + } as any); + + const res = await request(app) + .post('/api/admin/auth/signup') + .send({ username: 'admin', password: 'supersecretpassword123' }); + + expect(res.status).toBe(400); + expect(res.body.error).toContain('already been configured'); + }); + + it('should successfully configure admin and return 201', async () => { + vi.mocked(system_config_model.findOne).mockResolvedValue(null); + vi.mocked(system_config_model.create).mockResolvedValue({} as any); + + const res = await request(app) + .post('/api/admin/auth/signup') + .send({ username: 'superuser', password: 'supersecretpassword123' }); + + expect(res.status).toBe(201); + expect(res.body.success).toBe(true); + expect(res.body.token).toBeDefined(); + expect(typeof res.body.token).toBe('string'); + expect(res.body.user.username).toBe('superuser'); + expect(system_config_model.create).toHaveBeenCalled(); + expect(res.headers['set-cookie']).toBeDefined(); + expect(res.headers['set-cookie'].some((c: string) => c.includes('simplens_session='))).toBe(true); + }); + }); + + describe('POST /api/admin/auth/login', () => { + it('should reject invalid credentials', async () => { + vi.mocked(system_config_model.findOne).mockResolvedValue(null); + + const res = await request(app) + .post('/api/admin/auth/login') + .send({ username: 'wrong', password: 'wrongpassword' }); + + expect(res.status).toBe(401); + expect(res.body.isValid).toBe(false); + }); + + it('should verify correct credentials and return token', async () => { + const crypto = await import('crypto'); + const salt = crypto.randomBytes(32).toString('hex'); + const password_hash = crypto.scryptSync('supersecretpassword123', salt, 64).toString('hex'); + + vi.mocked(system_config_model.findOne).mockResolvedValue({ + key: 'admin_credentials', + value: { + username: 'admin', + password_hash, + salt, + created_at: new Date().toISOString(), + }, + } as any); + + const res = await request(app) + .post('/api/admin/auth/login') + .send({ username: 'admin', password: 'supersecretpassword123' }); + + expect(res.status).toBe(200); + expect(res.body.isValid).toBe(true); + expect(res.body.token).toBeDefined(); + expect(typeof res.body.token).toBe('string'); + expect(res.body.user.username).toBe('admin'); + expect(res.headers['set-cookie']).toBeDefined(); + expect(res.headers['set-cookie'].some((c: string) => c.includes('simplens_session='))).toBe(true); + }); + }); + + describe('POST /api/admin/auth/logout', () => { + it('should clear session cookie and return 200', async () => { + const res = await request(app).post('/api/admin/auth/logout'); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.headers['set-cookie']).toBeDefined(); + }); + }); +}); diff --git a/tests/integration/api/api-keys.test.ts b/tests/integration/api/api-keys.test.ts new file mode 100644 index 0000000..77cc81e --- /dev/null +++ b/tests/integration/api/api-keys.test.ts @@ -0,0 +1,392 @@ +/** + * Integration tests for API Key Management and Usage Tracking + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import crypto from 'crypto'; +import api_keys_router from '@src/api/routes/api-key.routes.js'; +import notification_router from '@src/api/routes/notification.routes.js'; +import admin_auth_router from '@src/api/routes/admin-auth.routes.js'; +import { auth_middleware } from '@src/api/middlewares/auth_middleware.js'; +import api_key_model from '@src/database/models/api-key.models.js'; +import notification_model from '@src/database/models/notification.models.js'; +import system_config_model from '@src/database/models/system-config.models.js'; +import { generateAdminJwt } from '@src/api/utils/jwt.utils.js'; + +// Mock models +vi.mock('@src/database/models/api-key.models.js', () => { + let mockKeys: any[] = []; + return { + default: { + find: vi.fn(() => ({ + sort: vi.fn().mockImplementation(() => Promise.resolve(mockKeys)), + })), + findOne: vi.fn((query: any) => { + if (query.key_hash) { + const match = mockKeys.find(k => k.key_hash === query.key_hash && (!query.status || k.status === query.status)); + return Promise.resolve(match ? { ...match, toObject: () => ({ ...match }) } : null); + } + if (query.key_id) { + const match = mockKeys.find(k => k.key_id === query.key_id); + return Promise.resolve(match ? { ...match, toObject: () => ({ ...match }) } : null); + } + return Promise.resolve(null); + }), + create: vi.fn((data: any) => { + const doc = { + ...data, + created_at: new Date(), + updated_at: new Date(), + toObject: () => ({ ...data, created_at: new Date(), updated_at: new Date() }), + }; + mockKeys.push(doc); + return Promise.resolve(doc); + }), + findOneAndUpdate: vi.fn((query: any, update: any) => { + const match = mockKeys.find(k => k.key_id === query.key_id); + if (!match) return Promise.resolve(null); + if (update.$set) { + Object.assign(match, update.$set); + } + return Promise.resolve({ ...match, toObject: () => ({ ...match }) }); + }), + updateOne: vi.fn((query: any, update: any) => { + const match = mockKeys.find(k => k.key_id === query.key_id); + if (match) { + if (update.$inc) { + for (const [field, val] of Object.entries(update.$inc)) { + if (field.startsWith('usage.by_channel.')) { + const ch = field.replace('usage.by_channel.', ''); + match.usage.by_channel[ch] = (match.usage.by_channel[ch] || 0) + (val as number); + } else if (field === 'usage.total_notifications') { + match.usage.total_notifications = (match.usage.total_notifications || 0) + (val as number); + } else if (field === 'usage.total_requests') { + match.usage.total_requests = (match.usage.total_requests || 0) + (val as number); + } + } + } + if (update.$set) { + Object.assign(match, update.$set); + } + } + return Promise.resolve({ modifiedCount: match ? 1 : 0 }); + }), + deleteOne: vi.fn((query: any) => { + const idx = mockKeys.findIndex(k => k.key_id === query.key_id); + if (idx >= 0) { + mockKeys.splice(idx, 1); + return Promise.resolve({ deletedCount: 1 }); + } + return Promise.resolve({ deletedCount: 0 }); + }), + __resetMockKeys: () => { + mockKeys = []; + }, + __getMockKeys: () => mockKeys, + }, + }; +}); + +vi.mock('@src/database/models/notification.models.js', () => ({ + default: { + aggregate: vi.fn().mockResolvedValue([ + { _id: 'delivered', count: 10 }, + { _id: 'failed', count: 2 }, + ]), + findOne: vi.fn().mockResolvedValue(null), + insertMany: vi.fn().mockResolvedValue([]), + }, +})); + +vi.mock('@src/database/models/outbox.models.js', () => ({ + default: { + insertMany: vi.fn().mockResolvedValue([]), + }, +})); + +vi.mock('@src/database/models/system-config.models.js', () => ({ + default: { + findOne: vi.fn().mockResolvedValue({ key: 'admin_credentials', value: { username: 'admin' } }), + }, +})); + +vi.mock('@src/plugins/index.js', () => { + const mockProviderInstance = { + getContentSchema: () => ({ + safeParse: () => ({ success: true, data: {} }), + }), + }; + + return { + PluginRegistry: { + getProvider: vi.fn().mockReturnValue(mockProviderInstance), + get: vi.fn().mockReturnValue(mockProviderInstance), + getDefaultProvider: vi.fn().mockReturnValue(mockProviderInstance), + hasChannel: vi.fn().mockReturnValue(true), + getChannelProviders: vi.fn().mockReturnValue(['mock-provider']), + }, + }; +}); + +vi.mock('@src/api/utils/utils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + process_notifications: vi.fn().mockResolvedValue({ + created_count: 1, + notification_ids: ['507f1f77bcf86cd799439011'], + duplicate_count: 0, + }), + }; +}); + +describe('API Keys Integration & Usage Tracking', () => { + let app: express.Application; + let adminToken: string; + + beforeEach(() => { + vi.clearAllMocks(); + (api_key_model as any).__resetMockKeys(); + + adminToken = generateAdminJwt({ id: 'admin-1', username: 'admin' }); + + app = express(); + app.use(express.json()); + app.use('/api/keys', auth_middleware, api_keys_router); + app.use('/api/notification', auth_middleware, notification_router); + app.use('/api/admin/auth', auth_middleware, admin_auth_router); + }); + + describe('API Key Management CRUD', () => { + it('should reject unauthenticated access to /api/keys', async () => { + const res = await request(app).get('/api/keys'); + expect(res.status).toBe(401); + }); + + it('should create an API key when authenticated with Admin JWT', async () => { + const res = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Payment Webhooks' }); + + expect(res.status).toBe(201); + expect(res.body.key).toBeDefined(); + expect(res.body.key.name).toBe('Payment Webhooks'); + expect(res.body.key.key_prefix).toMatch(/^sns_live_/); + expect(res.body.raw_key).toMatch(/^sns_live_/); + expect(res.body.key.status).toBe('active'); + }); + + it('should list API keys without exposing key_hash or raw key', async () => { + await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Service Alpha' }); + + const res = await request(app) + .get('/api/keys') + .set('Authorization', `Bearer ${adminToken}`); + + expect(res.status).toBe(200); + expect(res.body.keys).toHaveLength(1); + expect(res.body.keys[0].name).toBe('Service Alpha'); + expect(res.body.keys[0].key_hash).toBeUndefined(); + expect(res.body.keys[0].raw_key).toBeUndefined(); + expect(res.body.keys[0].key_prefix).toBeDefined(); + }); + + it('should get API key usage and delivery breakdown', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Analytics Key' }); + + const keyId = createRes.body.key.key_id; + + const res = await request(app) + .get(`/api/keys/${keyId}`) + .set('Authorization', `Bearer ${adminToken}`); + + expect(res.status).toBe(200); + expect(res.body.key.key_id).toBe(keyId); + expect(res.body.status_breakdown).toEqual({ + delivered: 10, + failed: 2, + pending: 0, + processing: 0, + }); + }); + + it('should revoke an API key', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Temporary Worker' }); + + const keyId = createRes.body.key.key_id; + + const revokeRes = await request(app) + .post(`/api/keys/${keyId}/revoke`) + .set('Authorization', `Bearer ${adminToken}`); + + expect(revokeRes.status).toBe(200); + expect(revokeRes.body.success).toBe(true); + expect(revokeRes.body.key.status).toBe('revoked'); + }); + + it('should delete an API key', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Obsolete Key' }); + + const keyId = createRes.body.key.key_id; + + const deleteRes = await request(app) + .delete(`/api/keys/${keyId}`) + .set('Authorization', `Bearer ${adminToken}`); + + expect(deleteRes.status).toBe(200); + expect(deleteRes.body.success).toBe(true); + + const getRes = await request(app) + .get(`/api/keys/${keyId}`) + .set('Authorization', `Bearer ${adminToken}`); + expect(getRes.status).toBe(404); + }); + }); + + describe('External Services Authentication & Usage Tracking', () => { + it('should authenticate external notification request via generated API key', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Order Notification Service' }); + + const rawKey = createRes.body.raw_key; + expect(rawKey).toBeDefined(); + + // Send notification using the generated API key + const notifPayload = { + request_id: crypto.randomUUID(), + client_id: crypto.randomUUID(), + channel: ['email'], + recipient: { email: 'customer@example.com' }, + content: { email: { subject: 'Receipt', message: 'Order paid' } }, + webhook_url: 'https://webhook.site/test', + }; + + const notifRes = await request(app) + .post('/api/notification') + .set('Authorization', `Bearer ${rawKey}`) + .send(notifPayload); + + expect(notifRes.status).toBe(202); + + // Verify usage tracking incremented + const keys = (api_key_model as any).__getMockKeys(); + const usedKey = keys[0]; + expect(usedKey.usage.total_notifications).toBeGreaterThanOrEqual(1); + expect(usedKey.usage.by_channel['email']).toBeGreaterThanOrEqual(1); + }); + + it('should reject requests made with a revoked API key', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'To Be Revoked' }); + + const keyId = createRes.body.key.key_id; + const rawKey = createRes.body.raw_key; + + // Revoke it + await request(app) + .post(`/api/keys/${keyId}/revoke`) + .set('Authorization', `Bearer ${adminToken}`); + + // Attempt to send notification + const notifRes = await request(app) + .post('/api/notification') + .set('Authorization', `Bearer ${rawKey}`) + .send({ + request_id: crypto.randomUUID(), + client_id: crypto.randomUUID(), + channel: ['email'], + recipient: { email: 'customer@example.com' }, + content: { email: { subject: 'Receipt', message: 'Order paid' } }, + webhook_url: 'https://webhook.site/test', + }); + + expect(notifRes.status).toBe(401); + }); + + it('should forbid API keys from accessing admin credential setup or verify routes', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'External Key' }); + + const rawKey = createRes.body.raw_key; + + const setupRes = await request(app) + .post('/api/admin/auth/setup') + .set('Authorization', `Bearer ${rawKey}`) + .send({ username: 'hacker', password: 'password123' }); + + expect(setupRes.status).toBe(403); + expect(setupRes.body.error).toContain('cannot access administrator credential'); + }); + + it('should allow API keys to access /api/admin/auth/status for MCP server compatibility', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'MCP Server Key' }); + + const rawKey = createRes.body.raw_key; + + const statusRes = await request(app) + .get('/api/admin/auth/status') + .set('Authorization', `Bearer ${rawKey}`); + + expect(statusRes.status).toBe(200); + expect(statusRes.body.isConfigured).toBe(true); + }); + + it('should forbid notification API keys from accessing /api/keys management route', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Worker Key', type: 'notification' }); + + const rawKey = createRes.body.raw_key; + expect(createRes.body.key.type).toBe('notification'); + + const keysRes = await request(app) + .get('/api/keys') + .set('Authorization', `Bearer ${rawKey}`); + + expect(keysRes.status).toBe(403); + expect(keysRes.body.error).toContain('management permissions'); + }); + + it('should allow admin API keys to access /api/keys management route', async () => { + const createRes = await request(app) + .post('/api/keys') + .set('Authorization', `Bearer ${adminToken}`) + .send({ name: 'Super Admin Key', type: 'admin' }); + + const rawKey = createRes.body.raw_key; + expect(createRes.body.key.type).toBe('admin'); + + const keysRes = await request(app) + .get('/api/keys') + .set('Authorization', `Bearer ${rawKey}`); + + expect(keysRes.status).toBe(200); + expect(keysRes.body.keys).toBeDefined(); + }); + }); +}); diff --git a/tests/integration/api/middlewares/auth_middleware.test.ts b/tests/integration/api/middlewares/auth_middleware.test.ts index f5c5de9..8fc7ee4 100644 --- a/tests/integration/api/middlewares/auth_middleware.test.ts +++ b/tests/integration/api/middlewares/auth_middleware.test.ts @@ -1,20 +1,23 @@ /** * Integration tests for auth_middleware.ts - * Tests API authentication middleware with mocked environment + * Tests API authentication middleware with Admin JWT and Dynamic API Keys */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import type { Request, Response, NextFunction } from 'express'; +import crypto from 'crypto'; +import { generateAdminJwt } from '../../../../src/api/utils/jwt.utils.js'; -// Mocked env that can be modified per test -let mockApiKey: string | undefined = 'valid-api-key-12345'; +// Mock DB model for api_key_model +const mockFindOne = vi.fn(); +const mockUpdateOne = vi.fn().mockReturnValue({ + catch: vi.fn(), +}); -// Mock env config - using a factory function to return fresh mock -vi.mock('@src/config/env.config.js', () => ({ - env: { - get NS_API_KEY() { - return mockApiKey; - }, +vi.mock('@src/database/models/api-key.models.js', () => ({ + default: { + findOne: (...args: unknown[]) => mockFindOne(...args), + updateOne: (...args: unknown[]) => mockUpdateOne(...args), }, })); @@ -31,12 +34,19 @@ vi.mock('@src/workers/utils/logger.js', () => ({ describe('Auth Middleware', () => { let authMiddleware: typeof import('../../../../src/api/middlewares/auth_middleware.js'); - // Helper to create mock request - const createMockRequest = (authHeader?: string): Partial => ({ - headers: authHeader ? { authorization: authHeader } : {}, + const createMockRequest = ( + authHeader?: string, + path: string = '/api/notifications', + cookieHeader?: string, + ): Partial => ({ + headers: { + ...(authHeader ? { authorization: authHeader } : {}), + ...(cookieHeader ? { cookie: cookieHeader } : {}), + }, + originalUrl: path, + path, }); - // Helper to create mock response const createMockResponse = (): Partial => { const res: Partial = { status: vi.fn().mockReturnThis(), @@ -47,11 +57,6 @@ describe('Auth Middleware', () => { beforeEach(async () => { vi.clearAllMocks(); - vi.resetModules(); - - // Reset to valid API key by default - mockApiKey = 'valid-api-key-12345'; - authMiddleware = await import('../../../../src/api/middlewares/auth_middleware.js'); }); @@ -59,158 +64,250 @@ describe('Auth Middleware', () => { vi.clearAllMocks(); }); - describe('Valid authentication', () => { - it('should call next() for valid API key', () => { - const req = createMockRequest('Bearer valid-api-key-12345'); - const res = createMockResponse(); + describe('Admin JWT Authentication', () => { + it('should call next() and populate req.user for valid Admin JWT', () => { + const token = generateAdminJwt({ id: 'admin-1', username: 'admin' }); + const req = createMockRequest(`Bearer ${token}`) as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); expect(next).toHaveBeenCalled(); expect(res.status).not.toHaveBeenCalled(); + expect(req.user?.username).toBe('admin'); }); }); - describe('Missing authentication', () => { - it('should return 401 when authorization header is missing', () => { - const req = createMockRequest(); - const res = createMockResponse(); + describe('Dynamic API Key Authentication', () => { + it('should call next() and populate req.apiKey for valid dynamic key', async () => { + const rawKey = 'sns_live_abcdef1234567890'; + const hashed = crypto.createHash('sha256').update(rawKey).digest('hex'); + + const mockKeyDoc = { + key_id: 'key-123', + name: 'Test Service', + key_hash: hashed, + status: 'active', + expires_at: null, + toObject: () => ({ + key_id: 'key-123', + name: 'Test Service', + status: 'active', + }), + }; + + mockFindOne.mockResolvedValueOnce(mockKeyDoc); + + const req = createMockRequest(`Bearer ${rawKey}`) as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + await authMiddleware.auth_middleware(req, res, next as NextFunction); + + expect(next).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect(req.apiKey?.name).toBe('Test Service'); + }); + + it('should return 401 when dynamic API key is not found or revoked', async () => { + mockFindOne.mockResolvedValueOnce(null); + + const req = createMockRequest('Bearer sns_live_unknownkey') as Request; + const res = createMockResponse() as Response; + const next = vi.fn(); + + await authMiddleware.auth_middleware(req, res, next as NextFunction); expect(res.status).toHaveBeenCalledWith(401); - expect(res.json).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('API KEY missing'), - }) - ); + expect(res.json).toHaveBeenCalledWith({ message: 'Invalid API KEY' }); expect(next).not.toHaveBeenCalled(); }); - it('should return 401 when Bearer prefix is missing', () => { - const req = createMockRequest('valid-api-key-12345'); - const res = createMockResponse(); + it('should return 401 when dynamic API key has expired', async () => { + const rawKey = 'sns_live_expiredkey'; + const mockKeyDoc = { + key_id: 'key-expired', + status: 'active', + expires_at: new Date(Date.now() - 10000), // past + toObject: () => ({ key_id: 'key-expired' }), + }; + + mockFindOne.mockResolvedValueOnce(mockKeyDoc); + + const req = createMockRequest(`Bearer ${rawKey}`) as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + await authMiddleware.auth_middleware(req, res, next as NextFunction); expect(res.status).toHaveBeenCalledWith(401); + expect(res.json).toHaveBeenCalledWith({ message: 'API key has expired' }); + expect(next).not.toHaveBeenCalled(); + }); + + it('should return 403 when dynamic API key accesses admin credential routes', async () => { + const rawKey = 'sns_live_testkey'; + const mockKeyDoc = { + key_id: 'key-123', + status: 'active', + expires_at: null, + toObject: () => ({ key_id: 'key-123' }), + }; + + mockFindOne.mockResolvedValueOnce(mockKeyDoc); + + const req = createMockRequest(`Bearer ${rawKey}`, '/api/admin/auth/setup') as Request; + const res = createMockResponse() as Response; + const next = vi.fn(); + + await authMiddleware.auth_middleware(req, res, next as NextFunction); + + expect(res.status).toHaveBeenCalledWith(403); + expect(next).not.toHaveBeenCalled(); + }); + + it('should return 403 when notification API key accesses management routes', async () => { + const rawKey = 'sns_live_notifkey'; + const mockKeyDoc = { + key_id: 'key-123', + status: 'active', + type: 'notification', + expires_at: null, + toObject: () => ({ key_id: 'key-123', type: 'notification' }), + }; + + mockFindOne.mockResolvedValueOnce(mockKeyDoc); + + const req = createMockRequest(`Bearer ${rawKey}`, '/api/providers') as Request; + const res = createMockResponse() as Response; + const next = vi.fn(); + + await authMiddleware.auth_middleware(req, res, next as NextFunction); + + expect(res.status).toHaveBeenCalledWith(403); expect(res.json).toHaveBeenCalledWith( expect.objectContaining({ - message: expect.stringContaining("'Bearer' is missing"), + error: expect.stringContaining('Requires admin key'), }) ); + expect(next).not.toHaveBeenCalled(); + }); + + it('should allow admin API key to access management routes', async () => { + const rawKey = 'sns_live_adminkey'; + const mockKeyDoc = { + key_id: 'key-admin', + status: 'active', + type: 'admin', + expires_at: null, + toObject: () => ({ key_id: 'key-admin', type: 'admin' }), + }; + + mockFindOne.mockResolvedValueOnce(mockKeyDoc); + + const req = createMockRequest(`Bearer ${rawKey}`, '/api/providers') as Request; + const res = createMockResponse() as Response; + const next = vi.fn(); + + await authMiddleware.auth_middleware(req, res, next as NextFunction); + + expect(next).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); }); }); - describe('Invalid authentication', () => { - it('should return 401 for invalid API key', () => { - const req = createMockRequest('Bearer wrong-api-key'); - const res = createMockResponse(); + describe('Missing or Invalid Authentication', () => { + it('should return 401 when authorization header is missing', () => { + const req = createMockRequest() as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); expect(res.status).toHaveBeenCalledWith(401); expect(res.json).toHaveBeenCalledWith( expect.objectContaining({ - message: 'Invalid API KEY', + message: expect.stringContaining('API KEY missing'), }) ); + expect(next).not.toHaveBeenCalled(); }); - it('should return 401 for API key with different length', () => { - const req = createMockRequest('Bearer short'); - const res = createMockResponse(); + it('should return 401 when Bearer prefix is missing', () => { + const req = createMockRequest('raw-string-without-bearer') as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); expect(res.status).toHaveBeenCalledWith(401); expect(res.json).toHaveBeenCalledWith( expect.objectContaining({ - message: 'Invalid API KEY', + message: expect.stringContaining("'Bearer' is missing"), }) ); + expect(next).not.toHaveBeenCalled(); }); - it('should return 401 for empty API key after Bearer', () => { - const req = createMockRequest('Bearer '); - const res = createMockResponse(); + it('should return 401 for unknown non-JWT non-dynamic token', () => { + const req = createMockRequest('Bearer random-legacy-token') as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - authMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); expect(res.status).toHaveBeenCalledWith(401); + expect(res.json).toHaveBeenCalledWith({ message: 'Invalid API KEY' }); + expect(next).not.toHaveBeenCalled(); }); }); - describe('Server configuration errors', () => { - it('should return 500 when NS_API_KEY is not configured', async () => { - mockApiKey = undefined; - vi.resetModules(); - const freshMiddleware = await import('../../../../src/api/middlewares/auth_middleware.js'); - - const req = createMockRequest('Bearer some-key'); - const res = createMockResponse(); + describe('Cookie Authentication', () => { + it('should authenticate via simplens_session cookie containing valid Admin JWT', () => { + const token = generateAdminJwt({ id: 'admin-1', username: 'admin' }); + const req = createMockRequest(undefined, '/api/notifications', `simplens_session=${token}`) as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - freshMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); - expect(res.status).toHaveBeenCalledWith(500); - expect(res.json).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'Server configuration error', - }) - ); + expect(next).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect(req.user?.username).toBe('admin'); }); - it('should return 500 when NS_API_KEY is empty string', async () => { - mockApiKey = ''; - vi.resetModules(); - const freshMiddleware = await import('../../../../src/api/middlewares/auth_middleware.js'); + it('should authenticate via simplens_session cookie among other cookies', () => { + const token = generateAdminJwt({ id: 'admin-1', username: 'admin' }); + const req = createMockRequest( + undefined, + '/api/notifications', + `theme=dark; simplens_session=${token}; lang=en` + ) as Request; + const res = createMockResponse() as Response; + const next = vi.fn(); + + authMiddleware.auth_middleware(req, res, next as NextFunction); - const req = createMockRequest('Bearer some-key'); - const res = createMockResponse(); + expect(next).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect(req.user?.username).toBe('admin'); + }); + + it('should return 401 when simplens_session cookie contains an invalid token', () => { + const req = createMockRequest( + undefined, + '/api/notifications', + 'simplens_session=invalid-cookie-token' + ) as Request; + const res = createMockResponse() as Response; const next = vi.fn(); - freshMiddleware.auth_middleware( - req as Request, - res as Response, - next as NextFunction - ); + authMiddleware.auth_middleware(req, res, next as NextFunction); - expect(res.status).toHaveBeenCalledWith(500); + expect(res.status).toHaveBeenCalledWith(401); + expect(res.json).toHaveBeenCalledWith({ message: 'Invalid API KEY' }); + expect(next).not.toHaveBeenCalled(); }); }); }); diff --git a/tests/integration/api/notification.controller.test.ts b/tests/integration/api/notification.controller.test.ts index 97c08f5..42ab80c 100644 --- a/tests/integration/api/notification.controller.test.ts +++ b/tests/integration/api/notification.controller.test.ts @@ -129,6 +129,7 @@ vi.mock("../../../src/api/utils/utils.js", () => { InvalidProviderChannelError, InvalidContentSchemaError, ProviderNotFoundError, + updateApiKeyNotificationUsage: vi.fn(), }; }); diff --git a/tests/integration/api/plugins.test.ts b/tests/integration/api/plugins.test.ts new file mode 100644 index 0000000..43c5179 --- /dev/null +++ b/tests/integration/api/plugins.test.ts @@ -0,0 +1,269 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import pluginsRouter from '../../../src/api/routes/plugins.routes.js'; +import { PluginManagerService, NpmAuthService } from '../../../src/plugins/index.js'; + +vi.mock('../../../src/plugins/index.js', () => ({ + PluginRegistry: { + isInitialized: vi.fn(() => true), + getPluginMetadata: vi.fn(() => ({ + channels: ['email'], + providers: { + email: [ + { + name: 'resend', + package: '@simplens/resend', + version: '1.0.0', + channel: 'email', + }, + ], + }, + counts: { channels: 1, providers: 1 }, + })), + }, + PluginManagerService: { + listInstalledPlugins: vi.fn(), + installPlugin: vi.fn(), + changePluginVersion: vi.fn(), + uninstallPlugin: vi.fn(), + }, + NpmAuthService: { + getAuthStatus: vi.fn(), + saveNpmAuth: vi.fn(), + deleteNpmAuth: vi.fn(), + }, +})); + +describe('Plugins API Integration', () => { + let app: express.Application; + + beforeEach(() => { + vi.clearAllMocks(); + app = express(); + app.use(express.json()); + app.use('/api/plugins', pluginsRouter); + }); + + describe('GET /api/plugins/npm-auth', () => { + it('should return npm auth status with registries list', async () => { + vi.mocked(NpmAuthService.getAuthStatus).mockResolvedValue({ + is_configured: true, + registries: [ + { + id: 'scope:@corp-a', + scope: '@corp-a', + registry_url: 'https://npm.corp-a.com/', + masked_token: 'npm_****1234', + updated_at: new Date().toISOString(), + }, + ], + masked_token: 'npm_****1234', + default_registry: 'https://registry.npmjs.org/', + }); + + const res = await request(app).get('/api/plugins/npm-auth'); + + expect(res.status).toBe(200); + expect(res.body.is_configured).toBe(true); + expect(res.body.registries).toHaveLength(1); + expect(res.body.registries[0].scope).toBe('@corp-a'); + expect(NpmAuthService.getAuthStatus).toHaveBeenCalledTimes(1); + }); + }); + + describe('POST /api/plugins/npm-auth', () => { + it('should save npm auth configuration with scope', async () => { + vi.mocked(NpmAuthService.saveNpmAuth).mockResolvedValue({ + is_configured: true, + registries: [], + masked_token: 'npm_****7890', + default_registry: 'https://npm.corp.internal/', + }); + + const res = await request(app) + .post('/api/plugins/npm-auth') + .send({ + token: 'npm_1234567890', + registry_url: 'https://npm.corp.internal/', + scope: '@myenterprise', + }); + + expect(res.status).toBe(200); + expect(res.body.status.is_configured).toBe(true); + expect(NpmAuthService.saveNpmAuth).toHaveBeenCalledWith( + 'npm_1234567890', + 'https://npm.corp.internal/', + '@myenterprise' + ); + }); + + it('should reject invalid payload when token is empty', async () => { + const res = await request(app) + .post('/api/plugins/npm-auth') + .send({ + token: '', + }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Bad Request'); + expect(res.body.details).toBeDefined(); + }); + }); + + describe('DELETE /api/plugins/npm-auth', () => { + it('should delete npm auth configuration for a specific registry id', async () => { + vi.mocked(NpmAuthService.deleteNpmAuth).mockResolvedValue(undefined); + + const res = await request(app).delete('/api/plugins/npm-auth?id=scope:@corp-a'); + + expect(res.status).toBe(200); + expect(res.body.message).toContain('scope:@corp-a'); + expect(NpmAuthService.deleteNpmAuth).toHaveBeenCalledWith('scope:@corp-a'); + }); + + it('should delete all npm auth configurations when no target specified', async () => { + vi.mocked(NpmAuthService.deleteNpmAuth).mockResolvedValue(undefined); + + const res = await request(app).delete('/api/plugins/npm-auth'); + + expect(res.status).toBe(200); + expect(res.body.message).toContain('removed successfully'); + expect(NpmAuthService.deleteNpmAuth).toHaveBeenCalledWith(undefined); + }); + }); + + describe('POST /api/plugins/install', () => { + it('should install plugin successfully without auth', async () => { + const mockPlugin = { + name: 'custom-email', + package: '@myorg/custom-email', + version: '1.0.0', + channel: 'email', + status: 'installed', + installed_at: new Date().toISOString(), + }; + vi.mocked(PluginManagerService.installPlugin).mockResolvedValue(mockPlugin as any); + + const res = await request(app) + .post('/api/plugins/install') + .send({ + package: '@myorg/custom-email', + version: '1.0.0', + }); + + expect(res.status).toBe(201); + expect(res.body.plugin).toEqual(mockPlugin); + expect(PluginManagerService.installPlugin).toHaveBeenCalledWith( + '@myorg/custom-email', + '1.0.0', + undefined + ); + }); + + it('should install private plugin with auth credentials provided', async () => { + const mockPlugin = { + name: 'private-slack', + package: '@enterprise/simplens-slack', + version: '2.0.0', + channel: 'slack', + status: 'installed', + installed_at: new Date().toISOString(), + }; + vi.mocked(PluginManagerService.installPlugin).mockResolvedValue(mockPlugin as any); + + const res = await request(app) + .post('/api/plugins/install') + .send({ + package: '@enterprise/simplens-slack', + version: '2.0.0', + auth: { + token: 'npm_secret_enterprise_token', + }, + }); + + expect(res.status).toBe(201); + expect(PluginManagerService.installPlugin).toHaveBeenCalledWith( + '@enterprise/simplens-slack', + '2.0.0', + { token: 'npm_secret_enterprise_token', save_token: true } + ); + }); + + it('should return 400 when package name is missing', async () => { + const res = await request(app) + .post('/api/plugins/install') + .send({ + version: '1.0.0', + }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Bad Request'); + expect(res.body.message).toContain('Invalid plugin installation payload'); + }); + + it('should return 400 when package is not a valid SimpleNS plugin', async () => { + vi.mocked(PluginManagerService.installPlugin).mockRejectedValue( + new Error("Package 'lodash' is not a valid SimpleNS plugin: Missing required provider methods: send, initialize") + ); + + const res = await request(app) + .post('/api/plugins/install') + .send({ + package: 'lodash', + }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Bad Request'); + expect(res.body.message).toContain('not a valid SimpleNS plugin'); + }); + + it('should return 400 when private package npm authentication fails', async () => { + vi.mocked(PluginManagerService.installPlugin).mockRejectedValue( + new Error("npm authentication failed (401/403) while installing '@private/pkg'. Configure npm auth token first.") + ); + + const res = await request(app) + .post('/api/plugins/install') + .send({ + package: '@private/pkg', + }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Bad Request'); + expect(res.body.message).toContain('npm authentication failed'); + }); + }); + + describe('GET /api/plugins/installed', () => { + it('should return list of installed plugins', async () => { + vi.mocked(PluginManagerService.listInstalledPlugins).mockResolvedValue([ + { + name: 'resend', + package: '@simplens/resend', + version: '1.0.0', + channel: 'email', + status: 'installed', + } as any, + ]); + + const res = await request(app).get('/api/plugins/installed'); + + expect(res.status).toBe(200); + expect(res.body.plugins).toHaveLength(1); + expect(res.body.plugins[0].package).toBe('@simplens/resend'); + }); + }); + + describe('DELETE /api/plugins/:package', () => { + it('should uninstall a plugin', async () => { + vi.mocked(PluginManagerService.uninstallPlugin).mockResolvedValue(undefined); + + const res = await request(app).delete('/api/plugins/@simplens/resend'); + + expect(res.status).toBe(200); + expect(res.body.message).toContain('uninstalled successfully'); + expect(PluginManagerService.uninstallPlugin).toHaveBeenCalledWith('@simplens/resend'); + }); + }); +}); diff --git a/tests/integration/api/settings.test.ts b/tests/integration/api/settings.test.ts new file mode 100644 index 0000000..0f25fb1 --- /dev/null +++ b/tests/integration/api/settings.test.ts @@ -0,0 +1,99 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import settings_router from '@src/api/routes/settings.routes.js'; +import { auth_middleware } from '@src/api/middlewares/auth_middleware.js'; +import { dynamicConfig } from '@src/config/dynamic-config.service.js'; +import { generateAdminJwt } from '@src/api/utils/jwt.utils.js'; + +vi.mock('@src/config/redis.config.js', () => ({ + getRedisClient: vi.fn(() => ({ + publish: vi.fn().mockResolvedValue(1), + duplicate: vi.fn(() => ({ + status: 'ready', + connect: vi.fn().mockResolvedValue(undefined), + subscribe: vi.fn().mockResolvedValue(undefined), + unsubscribe: vi.fn().mockResolvedValue(undefined), + quit: vi.fn().mockResolvedValue('OK'), + on: vi.fn(), + })), + })), +})); + +vi.mock('@src/database/models/system-config.models.js', () => ({ + default: { + findOne: vi.fn(), + findOneAndUpdate: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + }, +})); + +describe('Settings API Integration', () => { + let app: express.Application; + const adminToken = generateAdminJwt({ id: 'admin-1', username: 'admin' }); + + beforeEach(() => { + vi.clearAllMocks(); + app = express(); + app.use(express.json()); + app.use('/api/settings', auth_middleware, settings_router); + }); + + it('should reject unauthorized requests when bearer token is missing', async () => { + const res = await request(app).get('/api/settings'); + expect(res.status).toBe(401); + }); + + it('should return operational settings with valid authorization', async () => { + const res = await request(app) + .get('/api/settings') + .set('Authorization', `Bearer ${adminToken}`); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.settings.worker.outbox_batch_size).toBeDefined(); + }); + + it('should update operational settings with valid authorization', async () => { + const res = await request(app) + .put('/api/settings') + .set('Authorization', `Bearer ${adminToken}`) + .send({ + worker: { + outbox_batch_size: 150, + }, + logging: { + log_level: 'warn', + }, + }); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.settings.worker.outbox_batch_size).toBe(150); + expect(res.body.settings.logging.log_level).toBe('warn'); + }); + + it('should reject update with invalid schema bounds', async () => { + const res = await request(app) + .put('/api/settings') + .set('Authorization', `Bearer ${adminToken}`) + .send({ + worker: { + outbox_batch_size: 99999, // exceeds max 2000 + }, + }); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Validation failed'); + }); + + it('should reset operational settings to system defaults', async () => { + const res = await request(app) + .post('/api/settings/reset') + .set('Authorization', `Bearer ${adminToken}`); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.settings.worker.outbox_batch_size).toBe(100); + }); +}); diff --git a/tests/integration/mcp_server.test.ts b/tests/integration/mcp_server.test.ts index 5da2c43..de9b14c 100644 --- a/tests/integration/mcp_server.test.ts +++ b/tests/integration/mcp_server.test.ts @@ -103,9 +103,9 @@ describe('MCP Server Integration Tests', () => { await server.close(); }); - it('should register exactly 29 tools', async () => { + it('should register all expected tools', async () => { const toolsResult = await client.listTools(); - expect(toolsResult.tools.length).toBe(29); + expect(toolsResult.tools.length).toBe(54); const toolNames = toolsResult.tools.map(t => t.name); expect(toolNames).toContain('send_notification'); diff --git a/tests/integration/workers/consumers/status.consumer.test.ts b/tests/integration/workers/consumers/status.consumer.test.ts index 23d5a46..3d5360f 100644 --- a/tests/integration/workers/consumers/status.consumer.test.ts +++ b/tests/integration/workers/consumers/status.consumer.test.ts @@ -191,7 +191,7 @@ describe('Status Consumer', () => { expect.objectContaining({ status: 'delivered', }), - { new: true } + { returnDocument: 'after' } ); }); @@ -281,6 +281,46 @@ describe('Status Consumer', () => { ); }); + it('should update provider and provider_history in MongoDB when provided', async () => { + const history = [ + { + provider: 'primary', + status: 'rate_limited', + error_code: 'RATE_LIMITED', + error_message: 'Rate limit exceeded', + retryable: true, + attempted_at: new Date().toISOString(), + }, + { + provider: 'fallback', + status: 'delivered', + attempted_at: new Date().toISOString(), + duration_ms: 120, + } + ]; + const statusData = createStatusMessage({ + status: 'delivered', + provider: 'fallback', + provider_history: history, + }); + const kafkaMessage = createKafkaMessage(statusData); + + await capturedMessageHandler!(kafkaMessage); + + expect(mockFindByIdAndUpdate).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + status: 'delivered', + provider: 'fallback', + provider_history: expect.arrayContaining([ + expect.objectContaining({ provider: 'primary', status: 'rate_limited' }), + expect.objectContaining({ provider: 'fallback', status: 'delivered' }), + ]), + }), + expect.anything() + ); + }); + it('should send webhook callback when webhook_url is provided', async () => { const statusData = createStatusMessage({ webhook_url: 'https://example.com/webhook', diff --git a/tests/unit/api/health.test.ts b/tests/unit/api/health.test.ts new file mode 100644 index 0000000..81ace5a --- /dev/null +++ b/tests/unit/api/health.test.ts @@ -0,0 +1,180 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import mongoose from 'mongoose'; +import health_router from '@src/api/routes/health.routes.js'; +import { + getLiveness, + getReadiness, + setAppShuttingDown +} from '@src/api/controllers/health.controller.js'; +import { getRedisClient } from '@src/config/redis.config.js'; + +vi.mock('@src/config/redis.config.js', () => ({ + getRedisClient: vi.fn(), +})); + +describe('API Health Endpoints', () => { + let app: express.Application; + + beforeEach(() => { + vi.clearAllMocks(); + setAppShuttingDown(false); + + app = express(); + app.use('/api', health_router); + }); + + afterEach(() => { + setAppShuttingDown(false); + }); + + describe('GET /api/health', () => { + it('should return 200 and healthy when MongoDB and Redis are connected', async () => { + // Mock mongoose connection readyState = 1 (connected) + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(1); + + // Mock Redis status = 'ready' + vi.mocked(getRedisClient).mockReturnValue({ + status: 'ready', + } as unknown as ReturnType); + + const res = await request(app).get('/api/health'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + status: 'healthy', + timestamp: expect.any(String), + checks: { + mongodb: true, + redis: true, + }, + }); + }); + + it('should return 503 and unhealthy when MongoDB is disconnected', async () => { + // Mock mongoose connection readyState = 0 (disconnected) + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(0); + + vi.mocked(getRedisClient).mockReturnValue({ + status: 'ready', + } as unknown as ReturnType); + + const res = await request(app).get('/api/health'); + + expect(res.status).toBe(503); + expect(res.body).toEqual({ + status: 'unhealthy', + timestamp: expect.any(String), + checks: { + mongodb: false, + redis: true, + }, + }); + }); + + it('should return 503 and unhealthy when Redis is disconnected', async () => { + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(1); + + vi.mocked(getRedisClient).mockReturnValue({ + status: 'close', + } as unknown as ReturnType); + + const res = await request(app).get('/api/health'); + + expect(res.status).toBe(503); + expect(res.body).toEqual({ + status: 'unhealthy', + timestamp: expect.any(String), + checks: { + mongodb: true, + redis: false, + }, + }); + }); + + it('should return 503 when the app is shutting down', async () => { + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(1); + vi.mocked(getRedisClient).mockReturnValue({ + status: 'ready', + } as unknown as ReturnType); + + setAppShuttingDown(true); + + const res = await request(app).get('/api/health'); + + expect(res.status).toBe(503); + expect(res.body.status).toBe('unhealthy'); + expect(res.body.checks.mongodb).toBe(false); + expect(res.body.checks.redis).toBe(false); + }); + }); + + describe('GET /api/healthz (liveness)', () => { + it('should return 200 when process is not shutting down', async () => { + const res = await request(app).get('/api/healthz'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + service: 'api', + status: 'healthy', + timestamp: expect.any(String), + checks: { + process: true, + }, + }); + }); + + it('should return 503 when process is shutting down', async () => { + setAppShuttingDown(true); + + const res = await request(app).get('/api/healthz'); + + expect(res.status).toBe(503); + expect(res.body).toEqual({ + service: 'api', + status: 'unhealthy', + timestamp: expect.any(String), + checks: { + process: false, + }, + }); + }); + }); + + describe('GET /api/readyz (readiness)', () => { + it('should return 200 and ready when all checks pass', async () => { + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(1); + vi.mocked(getRedisClient).mockReturnValue({ + status: 'ready', + } as unknown as ReturnType); + + const res = await request(app).get('/api/readyz'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + service: 'api', + status: 'ready', + timestamp: expect.any(String), + checks: { + mongodb: true, + redis: true, + }, + }); + }); + + it('should return 503 when readiness checks fail', async () => { + vi.spyOn(mongoose.connection, 'readyState', 'get').mockReturnValue(0); + vi.mocked(getRedisClient).mockReturnValue({ + status: 'ready', + } as unknown as ReturnType); + + const res = await request(app).get('/api/readyz'); + + expect(res.status).toBe(503); + expect(res.body.status).toBe('not_ready'); + expect(res.body.checks.mongodb).toBe(false); + expect(res.body.checks.redis).toBe(true); + }); + }); +}); diff --git a/tests/unit/api/jwt.utils.test.ts b/tests/unit/api/jwt.utils.test.ts new file mode 100644 index 0000000..76ae252 --- /dev/null +++ b/tests/unit/api/jwt.utils.test.ts @@ -0,0 +1,67 @@ +/** + * Unit tests for jwt.utils.ts + */ + +import { describe, it, expect } from 'vitest'; +import { generateAdminJwt, verifyAdminJwt } from '@src/api/utils/jwt.utils.js'; + +describe('Admin JWT Utilities', () => { + it('should generate a valid JWT token for admin user', () => { + const admin = { id: 'admin-1', username: 'admin' }; + const token = generateAdminJwt(admin); + + expect(token).toBeDefined(); + expect(typeof token).toBe('string'); + expect(token.split('.')).toHaveLength(3); + }); + + it('should verify and decode a valid JWT token', () => { + const admin = { id: 'admin-42', username: 'superadmin' }; + const token = generateAdminJwt(admin); + + const decoded = verifyAdminJwt(token); + expect(decoded).not.toBeNull(); + expect(decoded?.sub).toBe('admin-42'); + expect(decoded?.username).toBe('superadmin'); + expect(decoded?.role).toBe('admin'); + }); + + it('should return null when token signature is tampered', () => { + const admin = { id: 'admin-1', username: 'admin' }; + const token = generateAdminJwt(admin); + const tamperedToken = token.slice(0, -5) + 'abcde'; + + const decoded = verifyAdminJwt(tamperedToken); + expect(decoded).toBeNull(); + }); + + it('should return null for malformed tokens', () => { + expect(verifyAdminJwt('not.a.valid.jwt')).toBeNull(); + expect(verifyAdminJwt('')).toBeNull(); + expect(verifyAdminJwt('random-string')).toBeNull(); + }); + + describe('JWT Secret Validation', () => { + it('should fail if JWT_SECRET is empty or whitespace', async () => { + const { validateJwtSecret } = await import('@src/config/env.config.js'); + expect(() => validateJwtSecret('')).toThrow(/missing/i); + expect(() => validateJwtSecret(' ')).toThrow(/missing/i); + }); + + it('should fail if JWT_SECRET is the default insecure placeholder', async () => { + const { validateJwtSecret, INSECURE_JWT_SECRET_FALLBACK } = await import('@src/config/env.config.js'); + expect(() => validateJwtSecret(INSECURE_JWT_SECRET_FALLBACK)).toThrow(/insecure placeholder/i); + }); + + it('should fail if JWT_SECRET is shorter than 32 characters', async () => { + const { validateJwtSecret } = await import('@src/config/env.config.js'); + expect(() => validateJwtSecret('short-secret-key-123')).toThrow(/too short/i); + }); + + it('should succeed when JWT_SECRET is at least 32 characters', async () => { + const { validateJwtSecret } = await import('@src/config/env.config.js'); + const valid = 'a-very-strong-and-secure-random-jwt-secret-key-32-chars'; + expect(validateJwtSecret(valid)).toBe(valid); + }); + }); +}); diff --git a/tests/unit/api/providers.controller.test.ts b/tests/unit/api/providers.controller.test.ts new file mode 100644 index 0000000..5a7887d --- /dev/null +++ b/tests/unit/api/providers.controller.test.ts @@ -0,0 +1,164 @@ +/** + * Unit Tests for Providers Controller Rate Limits Endpoints + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type { Request, Response } from 'express'; + +const mockGetAllProvidersRateLimitStatus = vi.fn(); +const mockGetProviderRateLimitStatus = vi.fn(); +const mockResetRateLimiter = vi.fn(); + +vi.mock('../../../src/processors/shared/rate-limiter.js', () => ({ + getAllProvidersRateLimitStatus: () => mockGetAllProvidersRateLimitStatus(), + getProviderRateLimitStatus: (id: string) => mockGetProviderRateLimitStatus(id), + resetRateLimiter: (id: string) => mockResetRateLimiter(id), +})); + +describe('Providers Controller - Rate Limits Endpoints', () => { + let mockReq: Partial; + let mockRes: Partial; + let jsonMock: ReturnType; + let statusMock: ReturnType; + + beforeEach(() => { + vi.clearAllMocks(); + jsonMock = vi.fn(); + statusMock = vi.fn().mockReturnThis(); + + mockReq = { + params: {}, + }; + mockRes = { + json: jsonMock, + status: statusMock, + }; + }); + + describe('getProvidersRateLimits', () => { + it('should return 200 with all providers rate limit statuses', async () => { + const { getProvidersRateLimits } = await import( + '../../../src/api/controllers/providers.controller.js' + ); + + const mockData = { + providers: [ + { + provider_id: 'gmail-1', + channel: 'email', + plugin_name: 'nodemailer-gmail', + enabled: true, + config: { max_tokens: 100, refill_rate: 10, refill_interval: 'second', normalized_refill_rate: 10 }, + status: { + remaining_tokens: 85, + used_tokens: 15, + usage_percentage: 15, + is_exhausted: false, + resets_in_ms: 0, + full_refill_in_ms: 1500, + last_refill_at: new Date(), + last_exhausted_at: null, + exhausted_count: 0, + }, + }, + ], + summary: { + total_providers: 1, + exhausted_providers: 0, + healthy_providers: 1, + }, + }; + + mockGetAllProvidersRateLimitStatus.mockResolvedValue(mockData); + + await getProvidersRateLimits(mockReq as Request, mockRes as Response); + + expect(jsonMock).toHaveBeenCalledWith(mockData); + }); + + it('should return 500 when retrieval fails', async () => { + const { getProvidersRateLimits } = await import( + '../../../src/api/controllers/providers.controller.js' + ); + + mockGetAllProvidersRateLimitStatus.mockRejectedValue(new Error('Redis connection failed')); + + await getProvidersRateLimits(mockReq as Request, mockRes as Response); + + expect(statusMock).toHaveBeenCalledWith(500); + expect(jsonMock).toHaveBeenCalledWith({ + error: 'Internal server error', + message: 'Failed to retrieve providers rate limits', + }); + }); + }); + + describe('getProviderRateLimit', () => { + it('should return 200 with specific provider rate limit', async () => { + const { getProviderRateLimit } = await import( + '../../../src/api/controllers/providers.controller.js' + ); + + mockReq.params = { id: 'gmail-1' }; + + const mockSingle = { + provider_id: 'gmail-1', + channel: 'email', + plugin_name: 'nodemailer-gmail', + enabled: true, + config: { max_tokens: 100, refill_rate: 10, refill_interval: 'second', normalized_refill_rate: 10 }, + status: { + remaining_tokens: 90, + used_tokens: 10, + usage_percentage: 10, + is_exhausted: false, + resets_in_ms: 0, + full_refill_in_ms: 1000, + last_refill_at: new Date(), + last_exhausted_at: null, + exhausted_count: 0, + }, + }; + + mockGetProviderRateLimitStatus.mockResolvedValue(mockSingle); + + await getProviderRateLimit(mockReq as Request, mockRes as Response); + + expect(jsonMock).toHaveBeenCalledWith({ rate_limit: mockSingle }); + }); + + it('should return 404 if provider rate limit status is not found', async () => { + const { getProviderRateLimit } = await import( + '../../../src/api/controllers/providers.controller.js' + ); + + mockReq.params = { id: 'unknown-provider' }; + mockGetProviderRateLimitStatus.mockResolvedValue(null); + + await getProviderRateLimit(mockReq as Request, mockRes as Response); + + expect(statusMock).toHaveBeenCalledWith(404); + expect(jsonMock).toHaveBeenCalledWith({ + error: 'Not Found', + message: "Provider 'unknown-provider' rate limit status not found", + }); + }); + }); + + describe('resetProviderRateLimit', () => { + it('should call resetRateLimiter and return success message', async () => { + const { resetProviderRateLimit } = await import( + '../../../src/api/controllers/providers.controller.js' + ); + + mockReq.params = { id: 'gmail-1' }; + mockResetRateLimiter.mockResolvedValue(undefined); + + await resetProviderRateLimit(mockReq as Request, mockRes as Response); + + expect(mockResetRateLimiter).toHaveBeenCalledWith('gmail-1'); + expect(jsonMock).toHaveBeenCalledWith({ + message: "Rate limit for provider 'gmail-1' has been reset successfully.", + }); + }); + }); +}); diff --git a/tests/unit/api/utils.test.ts b/tests/unit/api/utils.test.ts index e1e3201..38a5b01 100644 --- a/tests/unit/api/utils.test.ts +++ b/tests/unit/api/utils.test.ts @@ -38,6 +38,12 @@ vi.mock("../../../src/database/models/notification-template.models.js", () => ({ }, })); +vi.mock("../../../src/database/models/api-key.models.js", () => ({ + default: { + updateOne: vi.fn().mockReturnValue({ catch: vi.fn() }), + }, +})); + vi.mock("../../../src/workers/utils/logger.js", () => ({ apiLogger: { info: vi.fn(), @@ -1283,4 +1289,62 @@ describe("API Utility Functions", () => { expect(outbox_model.insertMany).not.toHaveBeenCalled(); }); }); + + describe("updateApiKeyNotificationUsage", () => { + it("should do nothing if apiKey is undefined", async () => { + const api_key_model = ( + await import("../../../src/database/models/api-key.models.js") + ).default; + const { updateApiKeyNotificationUsage } = + await import("../../../src/api/utils/utils.js"); + + updateApiKeyNotificationUsage(undefined, [], 5); + + expect(api_key_model.updateOne).not.toHaveBeenCalled(); + }); + + it("should do nothing if createdCount is 0 or negative", async () => { + const api_key_model = ( + await import("../../../src/database/models/api-key.models.js") + ).default; + const { updateApiKeyNotificationUsage } = + await import("../../../src/api/utils/utils.js"); + + const mockApiKey = { key_id: "ak_test123" } as any; + + updateApiKeyNotificationUsage(mockApiKey, [], 0); + updateApiKeyNotificationUsage(mockApiKey, [], -1); + + expect(api_key_model.updateOne).not.toHaveBeenCalled(); + }); + + it("should update key usage with total notifications and channel breakdown", async () => { + const api_key_model = ( + await import("../../../src/database/models/api-key.models.js") + ).default; + const { updateApiKeyNotificationUsage } = + await import("../../../src/api/utils/utils.js"); + + const mockApiKey = { key_id: "ak_test123" } as any; + const mockNotifications = [ + { channel: "email" }, + { channel: "email" }, + { channel: "sms" }, + ] as any[]; + + updateApiKeyNotificationUsage(mockApiKey, mockNotifications, 3); + + expect(api_key_model.updateOne).toHaveBeenCalledWith( + { key_id: "ak_test123" }, + expect.objectContaining({ + $set: expect.objectContaining({ "usage.last_used_at": expect.any(Date) }), + $inc: { + "usage.total_notifications": 3, + "usage.by_channel.email": 2, + "usage.by_channel.sms": 1, + }, + }), + ); + }); + }); }); diff --git a/tests/unit/api/webhook.controller.test.ts b/tests/unit/api/webhook.controller.test.ts new file mode 100644 index 0000000..14386dd --- /dev/null +++ b/tests/unit/api/webhook.controller.test.ts @@ -0,0 +1,40 @@ +import { describe, it, expect } from 'vitest'; +import request from 'supertest'; +import express from 'express'; +import webhook_router from '@src/api/routes/webhook.routes.js'; + +describe('Webhook Controller', () => { + let app: express.Application; + + beforeEach(() => { + app = express(); + app.use(express.json()); + app.use('/api/webhook', webhook_router); + }); + + it('should acknowledge valid webhook status callbacks with 200 and { received: true }', async () => { + const payload = { + request_id: 'req-123', + notification_id: 'notif-456', + status: 'delivered', + channel: 'email', + occurred_at: new Date().toISOString(), + }; + + const res = await request(app) + .post('/api/webhook') + .send(payload) + .expect(200); + + expect(res.body).toEqual({ received: true }); + }); + + it('should return 200 even for empty payload', async () => { + const res = await request(app) + .post('/api/webhook') + .send({}) + .expect(200); + + expect(res.body).toEqual({ received: true }); + }); +}); diff --git a/tests/unit/config/dynamic-config.test.ts b/tests/unit/config/dynamic-config.test.ts new file mode 100644 index 0000000..b4e38e8 --- /dev/null +++ b/tests/unit/config/dynamic-config.test.ts @@ -0,0 +1,142 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { + operationalSettingsSchema, + partialOperationalSettingsSchema, + adminSetupSchema, + adminLoginSchema, +} from '@src/types/schemas.js'; +import { + dynamicConfig, + SYSTEM_DEFAULT_OPERATIONAL_SETTINGS, +} from '@src/config/dynamic-config.service.js'; +import { env } from '@src/config/env.config.js'; + +vi.mock('@src/config/redis.config.js', () => ({ + getRedisClient: vi.fn(() => ({ + publish: vi.fn().mockResolvedValue(1), + duplicate: vi.fn(() => ({ + status: 'ready', + connect: vi.fn().mockResolvedValue(undefined), + subscribe: vi.fn().mockResolvedValue(undefined), + unsubscribe: vi.fn().mockResolvedValue(undefined), + quit: vi.fn().mockResolvedValue('OK'), + on: vi.fn(), + })), + })), +})); + +vi.mock('@src/database/models/system-config.models.js', () => ({ + default: { + findOne: vi.fn(), + findOneAndUpdate: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + }, +})); + +describe('Dynamic Operational Configuration', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe('Schema Validations', () => { + it('should validate SYSTEM_DEFAULT_OPERATIONAL_SETTINGS against schema', () => { + const parsed = operationalSettingsSchema.safeParse(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS); + expect(parsed.success).toBe(true); + }); + + it('should reject out-of-bound batch sizes', () => { + const invalid = { + ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS, + worker: { + ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS.worker, + outbox_batch_size: 0, // min is 1 + }, + }; + const parsed = operationalSettingsSchema.safeParse(invalid); + expect(parsed.success).toBe(false); + }); + + it('should reject invalid log levels', () => { + const invalid = { + ...SYSTEM_DEFAULT_OPERATIONAL_SETTINGS, + logging: { + log_level: 'verbose', // invalid enum + log_to_file: true, + }, + }; + const parsed = operationalSettingsSchema.safeParse(invalid); + expect(parsed.success).toBe(false); + }); + + it('should allow valid partial operational settings', () => { + const partial = { + worker: { + outbox_batch_size: 250, + }, + logging: { + log_level: 'debug' as const, + }, + }; + const parsed = partialOperationalSettingsSchema.safeParse(partial); + expect(parsed.success).toBe(true); + expect(parsed.data?.worker?.outbox_batch_size).toBe(250); + }); + + it('should validate admin setup requirements', () => { + expect(adminSetupSchema.safeParse({ username: 'ad', password: '123' }).success).toBe(false); + expect(adminSetupSchema.safeParse({ username: 'admin', password: 'short' }).success).toBe(false); + expect(adminSetupSchema.safeParse({ username: 'admin', password: 'securepassword123' }).success).toBe(true); + }); + + it('should validate admin login payload', () => { + expect(adminLoginSchema.safeParse({ username: '', password: '' }).success).toBe(false); + expect(adminLoginSchema.safeParse({ username: 'admin', password: 'password123' }).success).toBe(true); + }); + }); + + describe('DynamicConfigService and Env Getters', () => { + it('should return default settings initially', () => { + const all = dynamicConfig.getAll(); + expect(all.worker.outbox_batch_size).toBeDefined(); + expect(dynamicConfig.get('worker').outbox_batch_size).toBe(all.worker.outbox_batch_size); + }); + + it('should dynamically update settings and reflect via env getters', async () => { + let changeFired = false; + const listener = () => { + changeFired = true; + }; + dynamicConfig.on('change', listener); + + await dynamicConfig.updateSettings({ + worker: { + outbox_batch_size: 42, + }, + retry: { + rate_limit_retry_delay_ms: 3000, + }, + logging: { + log_level: 'debug', + }, + }); + + expect(dynamicConfig.get('worker').outbox_batch_size).toBe(42); + expect(dynamicConfig.get('retry').rate_limit_retry_delay_ms).toBe(3000); + expect(env.OUTBOX_BATCH_SIZE).toBe(42); + expect(env.RATE_LIMIT_RETRY_DELAY_MS).toBe(3000); + expect(env.LOG_LEVEL).toBe('debug'); + expect(changeFired).toBe(true); + + dynamicConfig.off('change', listener); + }); + + it('should reset settings to system defaults', async () => { + await dynamicConfig.resetSettings(); + expect(dynamicConfig.get('worker').outbox_batch_size).toBe(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS.worker.outbox_batch_size); + expect(dynamicConfig.get('retry').rate_limit_retry_delay_ms).toBe(5000); + expect(env.OUTBOX_BATCH_SIZE).toBe(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS.worker.outbox_batch_size); + expect(env.RATE_LIMIT_RETRY_DELAY_MS).toBe(5000); + expect(env.LOG_LEVEL).toBe(SYSTEM_DEFAULT_OPERATIONAL_SETTINGS.logging.log_level); + }); + }); +}); diff --git a/tests/unit/config/kafka-dynamic-topics.test.ts b/tests/unit/config/kafka-dynamic-topics.test.ts new file mode 100644 index 0000000..f47327c --- /dev/null +++ b/tests/unit/config/kafka-dynamic-topics.test.ts @@ -0,0 +1,173 @@ +/** + * Unit Tests for Dynamic Kafka Topic Management in kafka.config.ts + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { kafka, ensureChannelTopic, expandTopicPartitions } from '../../../src/config/kafka.config.js'; + +describe('Dynamic Kafka Topics', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + describe('ensureChannelTopic', () => { + it('should create missing channel topic with default 6 partitions', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + listTopics: vi.fn().mockResolvedValue([]), + createTopics: vi.fn().mockResolvedValue(true), + fetchTopicMetadata: vi.fn().mockResolvedValue({ topics: [] }), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + await ensureChannelTopic('email'); + + expect(adminMock.connect).toHaveBeenCalled(); + expect(adminMock.listTopics).toHaveBeenCalled(); + expect(adminMock.createTopics).toHaveBeenCalledWith({ + topics: [ + { + topic: 'email_notification', + numPartitions: 6, + replicationFactor: 1, + }, + ], + validateOnly: false, + timeout: 30000, + }); + expect(adminMock.disconnect).toHaveBeenCalled(); + }); + + it('should pass custom partition count to createTopics', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + listTopics: vi.fn().mockResolvedValue([]), + createTopics: vi.fn().mockResolvedValue(true), + fetchTopicMetadata: vi.fn().mockResolvedValue({ topics: [] }), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + await ensureChannelTopic('sms', 12); + + expect(adminMock.createTopics).toHaveBeenCalledWith({ + topics: [ + { + topic: 'sms_notification', + numPartitions: 12, + replicationFactor: 1, + }, + ], + validateOnly: false, + timeout: 30000, + }); + }); + + it('should not call admin.createTopics if topic already exists and partitions match', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + listTopics: vi.fn().mockResolvedValue(['email_notification']), + createTopics: vi.fn().mockResolvedValue(true), + fetchTopicMetadata: vi.fn().mockResolvedValue({ + topics: [ + { + name: 'email_notification', + partitions: new Array(6).fill({}), + }, + ], + }), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + await ensureChannelTopic('email', 6); + + expect(adminMock.createTopics).not.toHaveBeenCalled(); + }); + }); + + describe('expandTopicPartitions', () => { + it('should create topic if topic is not found in Kafka metadata', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + fetchTopicMetadata: vi.fn().mockResolvedValue({ + topics: [], + }), + createTopics: vi.fn().mockResolvedValue(true), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + const result = await expandTopicPartitions('push', 8); + + expect(adminMock.createTopics).toHaveBeenCalledWith({ + topics: [ + { + topic: 'push_notification', + numPartitions: 8, + replicationFactor: 1, + }, + ], + validateOnly: false, + timeout: 30000, + }); + expect(result).toEqual({ previous: 0, current: 8 }); + }); + + it('should expand partitions if requested count exceeds current count', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + fetchTopicMetadata: vi.fn().mockResolvedValue({ + topics: [ + { + name: 'email_notification', + partitions: new Array(6).fill({}), + }, + ], + }), + createPartitions: vi.fn().mockResolvedValue(true), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + const result = await expandTopicPartitions('email', 10); + + expect(adminMock.createPartitions).toHaveBeenCalledWith({ + topicPartitions: [ + { + topic: 'email_notification', + count: 10, + }, + ], + }); + expect(result).toEqual({ previous: 6, current: 10 }); + }); + + it('should throw error if attempting to decrease partitions', async () => { + const adminMock = { + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + fetchTopicMetadata: vi.fn().mockResolvedValue({ + topics: [ + { + name: 'email_notification', + partitions: new Array(10).fill({}), + }, + ], + }), + }; + + vi.spyOn(kafka, 'admin').mockReturnValue(adminMock as any); + + await expect(expandTopicPartitions('email', 4)).rejects.toThrow( + 'Kafka partition count cannot be decreased' + ); + }); + }); +}); diff --git a/tests/unit/plugins/channel-routing.service.test.ts b/tests/unit/plugins/channel-routing.service.test.ts new file mode 100644 index 0000000..ba02e1e --- /dev/null +++ b/tests/unit/plugins/channel-routing.service.test.ts @@ -0,0 +1,143 @@ +/** + * Unit Tests for ChannelRoutingService + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { ChannelRoutingService } from '../../../src/plugins/services/channel-routing.service.js'; +import ChannelRouting from '../../../src/database/models/channel-routing.models.js'; +import Provider from '../../../src/database/models/provider.models.js'; +import { PluginRegistry } from '../../../src/plugins/loader/registry.js'; +import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; +import * as kafkaConfig from '../../../src/config/kafka.config.js'; + +describe('ChannelRoutingService', () => { + beforeEach(() => { + vi.restoreAllMocks(); + PluginRegistry.clear(); + vi.spyOn(PluginSyncService, 'publish').mockResolvedValue(undefined); + }); + + describe('setChannelRouting', () => { + it('should throw error if default provider is not found', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue(null); + + await expect( + ChannelRoutingService.setChannelRouting('email', { + default_provider_id: 'nonexistent', + }) + ).rejects.toThrow("Default provider 'nonexistent' not found."); + }); + + it('should throw error if default provider belongs to a different channel', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ + id: 'sms-tw', + channel: 'sms', + } as any); + + await expect( + ChannelRoutingService.setChannelRouting('email', { + default_provider_id: 'sms-tw', + }) + ).rejects.toThrow( + "Default provider 'sms-tw' is configured for channel 'sms', not 'email'." + ); + }); + + it('should throw error if partitions are decreased', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ + id: 'email-primary', + channel: 'email', + } as any); + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue({ + channel: 'email', + partitions: 8, + } as any); + + await expect( + ChannelRoutingService.setChannelRouting('email', { + default_provider_id: 'email-primary', + partitions: 4, + }) + ).rejects.toThrow( + "Cannot decrease partitions for channel 'email' from 8 to 4. Kafka partitions can only be increased." + ); + }); + + it('should expand Kafka partitions when partition count is increased and update DB', async () => { + vi.spyOn(Provider, 'findOne').mockImplementation(async (query: any) => { + if (query.id === 'email-primary') { + return { id: 'email-primary', channel: 'email' } as any; + } + if (query.id === 'email-fb-1') { + return { id: 'email-fb-1', channel: 'email' } as any; + } + return null; + }); + + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue({ + channel: 'email', + partitions: 6, + } as any); + + const expandSpy = vi + .spyOn(kafkaConfig, 'expandTopicPartitions') + .mockResolvedValue(undefined); + + const updatedRouting = { + channel: 'email', + default_provider_id: 'email-primary', + fallback_provider_ids: ['email-fb-1'], + partitions: 12, + toObject: () => updatedRouting, + }; + + vi.spyOn(ChannelRouting, 'findOneAndUpdate').mockResolvedValue(updatedRouting as any); + const setConfigSpy = vi.spyOn(PluginRegistry, 'setChannelConfig'); + + const result = await ChannelRoutingService.setChannelRouting('email', { + default_provider_id: 'email-primary', + fallback_provider_ids: ['email-fb-1'], + partitions: 12, + }); + + expect(expandSpy).toHaveBeenCalledWith('email', 12); + expect(setConfigSpy).toHaveBeenCalledWith('email', { + default: 'email-primary', + fallback: ['email-fb-1'], + }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('CHANNEL_ROUTING_UPDATED', { + channel: 'email', + }); + expect(result.partitions).toBe(12); + }); + + it('should ensure Kafka topic is created when configuring new channel routing', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ + id: 'email-primary', + channel: 'email', + } as any); + + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue(null); + + const ensureTopicSpy = vi + .spyOn(kafkaConfig, 'ensureChannelTopic') + .mockResolvedValue(undefined); + + const updatedRouting = { + channel: 'email', + default_provider_id: 'email-primary', + fallback_provider_ids: [], + partitions: 6, + toObject: () => updatedRouting, + }; + + vi.spyOn(ChannelRouting, 'findOneAndUpdate').mockResolvedValue(updatedRouting as any); + + await ChannelRoutingService.setChannelRouting('email', { + default_provider_id: 'email-primary', + }); + + expect(ensureTopicSpy).toHaveBeenCalledWith('email', 6); + }); + }); +}); diff --git a/tests/unit/plugins/crypto.test.ts b/tests/unit/plugins/crypto.test.ts new file mode 100644 index 0000000..419ced0 --- /dev/null +++ b/tests/unit/plugins/crypto.test.ts @@ -0,0 +1,118 @@ +/** + * Unit Tests for Key-Pair Envelope Encryption Manager + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { + generateRSAKeyPair, + encryptCredentials, + decryptCredentials, + clearKeyPairCache, +} from '../../../src/plugins/crypto/keypair-manager.js'; +import system_config_model from '../../../src/database/models/system-config.models.js'; + +describe('Key-Pair Envelope Encryption Manager', () => { + beforeEach(() => { + clearKeyPairCache(); + vi.restoreAllMocks(); + }); + + describe('generateRSAKeyPair', () => { + it('should generate valid RSA 2048-bit public and private PEM keys', () => { + const keyPair = generateRSAKeyPair(); + expect(keyPair.publicKey).toContain('-----BEGIN PUBLIC KEY-----'); + expect(keyPair.publicKey).toContain('-----END PUBLIC KEY-----'); + expect(keyPair.privateKey).toContain('-----BEGIN PRIVATE KEY-----'); + expect(keyPair.privateKey).toContain('-----END PRIVATE KEY-----'); + }); + }); + + describe('encryptCredentials & decryptCredentials', () => { + it('should encrypt and decrypt credentials successfully with mock database', async () => { + const mockKeyPair = generateRSAKeyPair(); + + // Mock MongoDB system_config_model + vi.spyOn(system_config_model, 'findOne').mockImplementation(async (query: any) => { + if (query.key === 'plugin_credentials_public_key') { + return { key: 'plugin_credentials_public_key', value: mockKeyPair.publicKey } as any; + } + if (query.key === 'plugin_credentials_private_key') { + return { key: 'plugin_credentials_private_key', value: mockKeyPair.privateKey } as any; + } + return null; + }); + + const credentials = { + GMAIL_USER: 'test@gmail.com', + GMAIL_APP_PASSWORD: 'app-password-secret-123', + API_TOKEN: 'token_xyz_987654321', + }; + + const encrypted = await encryptCredentials(credentials); + + expect(encrypted).toHaveProperty('encrypted_data'); + expect(encrypted).toHaveProperty('iv'); + expect(encrypted).toHaveProperty('auth_tag'); + expect(encrypted).toHaveProperty('encrypted_dek'); + + expect(typeof encrypted.encrypted_data).toBe('string'); + expect(typeof encrypted.iv).toBe('string'); + expect(typeof encrypted.auth_tag).toBe('string'); + expect(typeof encrypted.encrypted_dek).toBe('string'); + + // Now decrypt + const decrypted = await decryptCredentials(encrypted); + expect(decrypted).toEqual(credentials); + }); + + it('should generate new keys and save them if not found in database or env', async () => { + const storedConfigs = new Map(); + + vi.spyOn(system_config_model, 'findOne').mockImplementation(async (query: any) => { + const val = storedConfigs.get(query.key); + return val ? ({ key: query.key, value: val } as any) : null; + }); + + vi.spyOn(system_config_model, 'findOneAndUpdate').mockImplementation(async (query: any, update: any) => { + storedConfigs.set(query.key, update.value); + return { key: query.key, value: update.value } as any; + }); + + const creds = { API_KEY: 'super_secret' }; + const encrypted = await encryptCredentials(creds); + + expect(storedConfigs.has('plugin_credentials_public_key')).toBe(true); + expect(storedConfigs.has('plugin_credentials_private_key')).toBe(true); + + const decrypted = await decryptCredentials(encrypted); + expect(decrypted).toEqual(creds); + }); + + it('should handle large credential payloads through hybrid AES envelope', async () => { + const mockKeyPair = generateRSAKeyPair(); + + vi.spyOn(system_config_model, 'findOne').mockImplementation(async (query: any) => { + if (query.key === 'plugin_credentials_public_key') { + return { key: 'plugin_credentials_public_key', value: mockKeyPair.publicKey } as any; + } + if (query.key === 'plugin_credentials_private_key') { + return { key: 'plugin_credentials_private_key', value: mockKeyPair.privateKey } as any; + } + return null; + }); + + // 10KB payload (exceeds direct RSA-2048 OAEP limit of ~214 bytes) + const largeCredentials = { + SERVICE_ACCOUNT_JSON: JSON.stringify({ + type: 'service_account', + private_key: '-----BEGIN PRIVATE KEY-----\n' + 'MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC...'.repeat(150) + '\n-----END PRIVATE KEY-----', + }), + }; + + const encrypted = await encryptCredentials(largeCredentials); + const decrypted = await decryptCredentials(encrypted); + + expect(decrypted).toEqual(largeCredentials); + }); + }); +}); diff --git a/tests/unit/plugins/npm-auth.service.test.ts b/tests/unit/plugins/npm-auth.service.test.ts new file mode 100644 index 0000000..829d108 --- /dev/null +++ b/tests/unit/plugins/npm-auth.service.test.ts @@ -0,0 +1,367 @@ +/** + * Unit Tests for NpmAuthService + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { + NpmAuthService, + extractScope, + toNpmrcTokenLine, + normalizeRegistryUrl, + generateRegistryId, + maskToken, + NPM_AUTH_CONFIG_KEY, + NPMRC_PATH, +} from '../../../src/plugins/services/npm-auth.service.js'; +import system_config_model from '../../../src/database/models/system-config.models.js'; +import * as keyManager from '../../../src/admin-alerts/key-manager.js'; +import { encrypt } from '../../../src/utils/encryption.utils.js'; +import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; +import * as fs from 'fs'; + +vi.mock('fs', async () => { + const actual = await vi.importActual('fs'); + return { + ...actual, + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), + existsSync: vi.fn().mockReturnValue(true), + }; +}); + +describe('NpmAuthService', () => { + const mockEncryptionKey = Buffer.alloc(32, 7); + + beforeEach(() => { + vi.restoreAllMocks(); + vi.spyOn(keyManager, 'getOrCreateEncryptionKey').mockResolvedValue(mockEncryptionKey); + vi.spyOn(PluginSyncService, 'publish').mockResolvedValue(undefined); + }); + + describe('extractScope', () => { + it('should extract scope from scoped package name and lowercase it', () => { + expect(extractScope('@simplens/resend')).toBe('@simplens'); + expect(extractScope('@MyEnterprise/custom-sms')).toBe('@myenterprise'); + expect(extractScope('@org-name/sub-pkg')).toBe('@org-name'); + }); + + it('should return undefined for unscoped package names', () => { + expect(extractScope('nodemailer-gmail')).toBeUndefined(); + expect(extractScope('custom-plugin')).toBeUndefined(); + }); + }); + + describe('toNpmrcTokenLine', () => { + it('should correctly format standard npmjs URL', () => { + const line = toNpmrcTokenLine('https://registry.npmjs.org/', 'npm_test123'); + expect(line).toBe('//registry.npmjs.org/:_authToken=npm_test123'); + }); + + it('should correctly format enterprise registry URL', () => { + const line = toNpmrcTokenLine('https://npm.enterprise.corp/', 'npm_corp_token'); + expect(line).toBe('//npm.enterprise.corp/:_authToken=npm_corp_token'); + }); + + it('should handle URLs without trailing slash', () => { + const line = toNpmrcTokenLine('https://npm.enterprise.corp', 'npm_corp_token'); + expect(line).toBe('//npm.enterprise.corp/:_authToken=npm_corp_token'); + }); + }); + + describe('normalizeRegistryUrl & generateRegistryId & maskToken', () => { + it('should normalize URLs with trailing slash', () => { + expect(normalizeRegistryUrl('https://npm.corp.com')).toBe('https://npm.corp.com/'); + expect(normalizeRegistryUrl('')).toBe('https://registry.npmjs.org/'); + }); + + it('should generate consistent registry identifiers', () => { + expect(generateRegistryId('@myorg')).toBe('scope:@myorg'); + expect(generateRegistryId(undefined, 'https://npm.corp.com/')).toBe('url:npm.corp.com'); + expect(generateRegistryId(undefined, 'https://registry.npmjs.org/')).toBe('default'); + }); + + it('should mask tokens properly', () => { + expect(maskToken('npm_1234567890abcdef')).toBe('npm_****cdef'); + expect(maskToken('short')).toBe('****'); + }); + }); + + describe('getAuthStatus & getDecryptedRegistries', () => { + it('should return is_configured false when no registries exist and no ENV var exists', async () => { + delete process.env.NPM_TOKEN; + delete process.env.NPM_AUTH_TOKEN; + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue(null), + } as any); + + const status = await NpmAuthService.getAuthStatus(); + expect(status.is_configured).toBe(false); + expect(status.registries).toHaveLength(0); + }); + + it('should migrate legacy single-token document and return status with masked token', async () => { + const plainToken = 'npm_1234567890abcdef123456'; + const encrypted = encrypt(plainToken, mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + encrypted_token: encrypted, + registry_url: 'https://registry.npmjs.org/', + }, + }), + } as any); + + const status = await NpmAuthService.getAuthStatus(); + expect(status.is_configured).toBe(true); + expect(status.registries).toHaveLength(1); + expect(status.registries[0].masked_token).toContain('****'); + expect(status.registries[0].masked_token.startsWith('npm_')).toBe(true); + }); + + it('should return multiple configured registries with masked tokens', async () => { + const tokenA = encrypt('npm_corp_aaa_token_1234', mockEncryptionKey); + const tokenB = encrypt('npm_corp_bbb_token_5678', mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [ + { + id: 'scope:@corp-a', + scope: '@corp-a', + registry_url: 'https://npm.corp-a.com/', + encrypted_token: tokenA, + updated_at: new Date().toISOString(), + }, + { + id: 'scope:@corp-b', + scope: '@corp-b', + registry_url: 'https://npm.corp-b.com/', + encrypted_token: tokenB, + updated_at: new Date().toISOString(), + }, + ], + }, + }), + } as any); + + const status = await NpmAuthService.getAuthStatus(); + expect(status.is_configured).toBe(true); + expect(status.registries).toHaveLength(2); + expect(status.registries[0].scope).toBe('@corp-a'); + expect(status.registries[1].scope).toBe('@corp-b'); + expect(status.registries[0].masked_token).toBe('npm_****1234'); + expect(status.registries[1].masked_token).toBe('npm_****5678'); + }); + }); + + describe('findMatchingAuth', () => { + it('should match by scope when package name matches a registered scope', async () => { + const tokenA = encrypt('token_for_corp_a', mockEncryptionKey); + const tokenB = encrypt('token_for_corp_b', mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [ + { + id: 'scope:@corp-a', + scope: '@corp-a', + registry_url: 'https://npm.corp-a.com/', + encrypted_token: tokenA, + updated_at: new Date().toISOString(), + }, + { + id: 'scope:@corp-b', + scope: '@corp-b', + registry_url: 'https://npm.corp-b.com/', + encrypted_token: tokenB, + updated_at: new Date().toISOString(), + }, + ], + }, + }), + } as any); + + const match = await NpmAuthService.findMatchingAuth('@corp-a/email-provider'); + expect(match).not.toBeNull(); + expect(match?.scope).toBe('@corp-a'); + expect(match?.token).toBe('token_for_corp_a'); + expect(match?.registry_url).toBe('https://npm.corp-a.com/'); + }); + + it('should match by registry host when custom registry URL is specified', async () => { + const tokenCustom = encrypt('token_for_nexus', mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [ + { + id: 'url:nexus.internal.com', + registry_url: 'https://nexus.internal.com/repository/npm/', + encrypted_token: tokenCustom, + updated_at: new Date().toISOString(), + }, + ], + }, + }), + } as any); + + const match = await NpmAuthService.findMatchingAuth('unscoped-pkg', 'https://nexus.internal.com/repository/npm/'); + expect(match).not.toBeNull(); + expect(match?.token).toBe('token_for_nexus'); + }); + + it('should return null when no credentials match', async () => { + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue(null), + } as any); + + const match = await NpmAuthService.findMatchingAuth('@unregistered/package'); + expect(match).toBeNull(); + }); + }); + + describe('saveNpmAuth & deleteNpmAuth', () => { + it('should append or update registry entry in MongoDB and broadcast event', async () => { + const upsertSpy = vi.spyOn(system_config_model, 'findOneAndUpdate').mockResolvedValue({} as any); + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [], + }, + }), + } as any); + vi.spyOn(NpmAuthService, 'syncNpmrc').mockResolvedValue(undefined); + + await NpmAuthService.saveNpmAuth('npm_enterprise_token', 'https://npm.enterprise.corp/', '@myorg'); + + expect(upsertSpy).toHaveBeenCalledWith( + { key: NPM_AUTH_CONFIG_KEY }, + expect.objectContaining({ + key: NPM_AUTH_CONFIG_KEY, + value: expect.objectContaining({ + registries: expect.arrayContaining([ + expect.objectContaining({ + id: 'scope:@myorg', + scope: '@myorg', + registry_url: 'https://npm.enterprise.corp/', + }), + ]), + }), + }), + { upsert: true, new: true } + ); + expect(PluginSyncService.publish).toHaveBeenCalledWith('NPM_AUTH_UPDATED', {}); + }); + + it('should delete specific registry by scope or id', async () => { + const upsertSpy = vi.spyOn(system_config_model, 'findOneAndUpdate').mockResolvedValue({} as any); + const tokenA = encrypt('tokenA', mockEncryptionKey); + const tokenB = encrypt('tokenB', mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [ + { + id: 'scope:@corp-a', + scope: '@corp-a', + registry_url: 'https://npm.corp-a.com/', + encrypted_token: tokenA, + updated_at: new Date().toISOString(), + }, + { + id: 'scope:@corp-b', + scope: '@corp-b', + registry_url: 'https://npm.corp-b.com/', + encrypted_token: tokenB, + updated_at: new Date().toISOString(), + }, + ], + }, + }), + } as any); + vi.spyOn(NpmAuthService, 'syncNpmrc').mockResolvedValue(undefined); + + await NpmAuthService.deleteNpmAuth('@corp-a'); + + expect(upsertSpy).toHaveBeenCalledWith( + { key: NPM_AUTH_CONFIG_KEY }, + expect.objectContaining({ + value: expect.objectContaining({ + registries: [ + expect.objectContaining({ + id: 'scope:@corp-b', + }), + ], + }), + }), + { upsert: true, new: true } + ); + expect(PluginSyncService.publish).toHaveBeenCalledWith('NPM_AUTH_UPDATED', {}); + }); + }); + + describe('syncNpmrc', () => { + it('should write .npmrc with multiple scoped registries and host auth lines', async () => { + const tokenA = encrypt('token_corp_a', mockEncryptionKey); + const tokenB = encrypt('token_corp_b', mockEncryptionKey); + + vi.spyOn(system_config_model, 'findOne').mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + key: NPM_AUTH_CONFIG_KEY, + value: { + registries: [ + { + id: 'scope:@corp-a', + scope: '@corp-a', + registry_url: 'https://npm.corp-a.com/', + encrypted_token: tokenA, + updated_at: new Date().toISOString(), + }, + { + id: 'scope:@corp-b', + scope: '@corp-b', + registry_url: 'https://npm.corp-b.com/', + encrypted_token: tokenB, + updated_at: new Date().toISOString(), + }, + ], + }, + }), + } as any); + + await NpmAuthService.syncNpmrc(); + + expect(fs.writeFileSync).toHaveBeenCalledWith( + NPMRC_PATH, + expect.stringContaining('@corp-a:registry=https://npm.corp-a.com/'), + expect.anything() + ); + expect(fs.writeFileSync).toHaveBeenCalledWith( + NPMRC_PATH, + expect.stringContaining('//npm.corp-a.com/:_authToken=token_corp_a'), + expect.anything() + ); + expect(fs.writeFileSync).toHaveBeenCalledWith( + NPMRC_PATH, + expect.stringContaining('@corp-b:registry=https://npm.corp-b.com/'), + expect.anything() + ); + expect(fs.writeFileSync).toHaveBeenCalledWith( + NPMRC_PATH, + expect.stringContaining('//npm.corp-b.com/:_authToken=token_corp_b'), + expect.anything() + ); + }); + }); +}); diff --git a/tests/unit/plugins/plugin-fs.test.ts b/tests/unit/plugins/plugin-fs.test.ts new file mode 100644 index 0000000..a5da289 --- /dev/null +++ b/tests/unit/plugins/plugin-fs.test.ts @@ -0,0 +1,92 @@ +/** + * Unit tests for plugin-fs.ts + * Tests package name and version validation and spawnSync command execution + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockSpawnSync = vi.fn(); +vi.mock('child_process', () => ({ + spawnSync: (...args: unknown[]) => mockSpawnSync(...args), + readFileSync: vi.fn(), + existsSync: vi.fn().mockReturnValue(true), + mkdirSync: vi.fn(), + writeFileSync: vi.fn(), +})); + +describe('Plugin FS Package Utilities', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockSpawnSync.mockReturnValue({ + status: 0, + stderr: Buffer.from(''), + stdout: Buffer.from(''), + }); + }); + + describe('installNpmPackage', () => { + it('should reject package names with shell injection characters', async () => { + const { installNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + expect(() => installNpmPackage('express; rm -rf /')).toThrow(/invalid npm package name/i); + expect(() => installNpmPackage('express && calc')).toThrow(/invalid npm package name/i); + expect(() => installNpmPackage('express | whoami')).toThrow(/invalid npm package name/i); + expect(() => installNpmPackage('`calc`')).toThrow(/invalid npm package name/i); + expect(() => installNpmPackage('$(whoami)')).toThrow(/invalid npm package name/i); + expect(() => installNpmPackage('foo/bar/baz')).toThrow(/invalid npm package name/i); + expect(mockSpawnSync).not.toHaveBeenCalled(); + }); + + it('should reject version strings with shell injection characters', async () => { + const { installNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + expect(() => installNpmPackage('lodash', '1.0.0; calc')).toThrow(/invalid npm package version/i); + expect(() => installNpmPackage('lodash', '^1.0.0 & whoami')).toThrow(/invalid npm package version/i); + expect(() => installNpmPackage('lodash', '$(cat /etc/passwd)')).toThrow(/invalid npm package version/i); + expect(mockSpawnSync).not.toHaveBeenCalled(); + }); + + it('should invoke spawnSync with argument array for valid packages', async () => { + const { installNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + installNpmPackage('lodash', '4.17.21'); + + expect(mockSpawnSync).toHaveBeenCalledTimes(1); + const [cmd, args] = mockSpawnSync.mock.calls[0]; + expect(cmd).toMatch(/npm(\.cmd)?$/); + expect(args).toEqual(['install', 'lodash@4.17.21']); + }); + + it('should invoke spawnSync with argument array for valid scoped packages', async () => { + const { installNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + installNpmPackage('@simplens/provider-smtp', '^1.2.0'); + + expect(mockSpawnSync).toHaveBeenCalledTimes(1); + const [cmd, args] = mockSpawnSync.mock.calls[0]; + expect(cmd).toMatch(/npm(\.cmd)?$/); + expect(args).toEqual(['install', '@simplens/provider-smtp@^1.2.0']); + }); + }); + + describe('uninstallNpmPackage', () => { + it('should reject package names with shell injection characters', async () => { + const { uninstallNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + expect(() => uninstallNpmPackage('lodash; rm -rf /')).toThrow(/invalid npm package name/i); + expect(() => uninstallNpmPackage('foo & calc')).toThrow(/invalid npm package name/i); + expect(mockSpawnSync).not.toHaveBeenCalled(); + }); + + it('should invoke spawnSync with argument array for valid package', async () => { + const { uninstallNpmPackage } = await import('@src/plugins/loader/plugin-fs.js'); + + uninstallNpmPackage('@simplens/provider-smtp'); + + expect(mockSpawnSync).toHaveBeenCalledTimes(1); + const [cmd, args] = mockSpawnSync.mock.calls[0]; + expect(cmd).toMatch(/npm(\.cmd)?$/); + expect(args).toEqual(['uninstall', '@simplens/provider-smtp']); + }); + }); +}); diff --git a/tests/unit/plugins/plugin-manager.service.test.ts b/tests/unit/plugins/plugin-manager.service.test.ts new file mode 100644 index 0000000..9be20c6 --- /dev/null +++ b/tests/unit/plugins/plugin-manager.service.test.ts @@ -0,0 +1,187 @@ +/** + * Unit Tests for PluginManagerService + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { PluginManagerService } from '../../../src/plugins/services/plugin-manager.service.js'; +import Plugin from '../../../src/database/models/plugin.models.js'; +import Provider from '../../../src/database/models/provider.models.js'; +import ChannelRouting from '../../../src/database/models/channel-routing.models.js'; +import { PluginRegistry } from '../../../src/plugins/loader/registry.js'; +import * as pluginFs from '../../../src/plugins/loader/plugin-fs.js'; +import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; + +describe('PluginManagerService', () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.spyOn(PluginSyncService, 'publish').mockResolvedValue(undefined); + }); + + describe('installPlugin', () => { + it('should install npm package, extract manifest with package.json version, save to DB, and publish event', async () => { + const installSpy = vi.spyOn(pluginFs, 'installNpmPackage').mockReturnValue(undefined); + vi.spyOn(pluginFs, 'extractPackageManifest').mockResolvedValue({ + manifest: { + name: '@simplens/mock', + displayName: 'Mock Provider', + version: '2.1.0', + channel: 'mock', + description: 'Mock plugin', + requiredCredentials: [], + }, + version: '2.1.0', + }); + + const mockPluginDoc = { + name: '@simplens/mock', + version: '2.1.0', + status: 'installed', + manifest: { + name: '@simplens/mock', + displayName: 'Mock Provider', + version: '2.1.0', + channel: 'mock', + description: 'Mock plugin', + requiredCredentials: [], + }, + toObject: () => ({ + name: '@simplens/mock', + version: '2.1.0', + status: 'installed', + }), + }; + + vi.spyOn(Plugin, 'findOneAndUpdate').mockResolvedValue(mockPluginDoc as any); + + const result = await PluginManagerService.installPlugin('@simplens/mock', '2.1.0'); + + expect(installSpy).toHaveBeenCalledWith('@simplens/mock', '2.1.0'); + expect(Plugin.findOneAndUpdate).toHaveBeenCalledWith( + { name: '@simplens/mock' }, + expect.objectContaining({ + name: '@simplens/mock', + version: '2.1.0', + status: 'installed', + }), + { upsert: true, new: true } + ); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PLUGIN_INSTALLED', { + plugin_name: '@simplens/mock', + version: '2.1.0', + }); + expect(result.version).toBe('2.1.0'); + }); + + it('should roll back by uninstalling if manifest extraction/validation fails', async () => { + vi.spyOn(pluginFs, 'installNpmPackage').mockReturnValue(undefined); + const uninstallSpy = vi.spyOn(pluginFs, 'uninstallNpmPackage').mockReturnValue(undefined); + vi.spyOn(pluginFs, 'extractPackageManifest').mockRejectedValue( + new Error("Package 'lodash' is not a valid SimpleNS plugin.") + ); + + await expect( + PluginManagerService.installPlugin('lodash', 'latest') + ).rejects.toThrow("Package 'lodash' is not a valid SimpleNS plugin."); + + expect(uninstallSpy).toHaveBeenCalledWith('lodash'); + }); + }); + + describe('changePluginVersion', () => { + it('should throw error if plugin is not already installed', async () => { + vi.spyOn(Plugin, 'findOne').mockResolvedValue(null); + + await expect( + PluginManagerService.changePluginVersion('@simplens/unknown', '1.2.0') + ).rejects.toThrow("Plugin '@simplens/unknown' is not installed."); + }); + + it('should install new version and update DB when plugin exists', async () => { + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ name: '@simplens/mock' } as any); + vi.spyOn(pluginFs, 'installNpmPackage').mockReturnValue(undefined); + vi.spyOn(pluginFs, 'extractPackageManifest').mockResolvedValue({ + manifest: { + name: '@simplens/mock', + displayName: 'Mock Provider', + version: '3.0.0', + channel: 'mock', + description: 'Mock plugin v3', + requiredCredentials: [], + }, + version: '3.0.0', + }); + + const updatedDoc = { + name: '@simplens/mock', + version: '3.0.0', + toObject: () => ({ + name: '@simplens/mock', + version: '3.0.0', + }), + }; + vi.spyOn(Plugin, 'findOneAndUpdate').mockResolvedValue(updatedDoc as any); + + const result = await PluginManagerService.changePluginVersion('@simplens/mock', '3.0.0'); + + expect(PluginSyncService.publish).toHaveBeenCalledWith('PLUGIN_VERSION_CHANGED', { + plugin_name: '@simplens/mock', + version: '3.0.0', + }); + expect(result.version).toBe('3.0.0'); + }); + }); + + describe('uninstallPlugin', () => { + it('should remove dependent routings and providers before uninstalling the plugin', async () => { + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ name: '@simplens/mock' } as any); + vi.spyOn(Provider, 'find').mockReturnValue({ + select: () => ({ + lean: vi.fn().mockResolvedValue([{ id: 'mock-provider' }]), + }), + } as any); + vi.spyOn(ChannelRouting, 'find').mockReturnValue({ + select: () => ({ + lean: vi.fn().mockResolvedValue([{ channel: 'mock' }]), + }), + } as any); + const routingDeleteSpy = vi.spyOn(ChannelRouting, 'deleteMany').mockResolvedValue({ deletedCount: 1 } as any); + const providerDeleteSpy = vi.spyOn(Provider, 'deleteMany').mockResolvedValue({ deletedCount: 1 } as any); + const unregisterSpy = vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); + const uninstallSpy = vi.spyOn(pluginFs, 'uninstallNpmPackage').mockReturnValue(undefined); + const deleteSpy = vi.spyOn(Plugin, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); + + await PluginManagerService.uninstallPlugin('@simplens/mock'); + + expect(uninstallSpy).toHaveBeenCalledWith('@simplens/mock'); + expect(routingDeleteSpy).toHaveBeenCalledWith({ channel: { $in: ['mock'] } }); + expect(providerDeleteSpy).toHaveBeenCalledWith({ plugin_name: '@simplens/mock' }); + expect(unregisterSpy).toHaveBeenCalledWith('mock-provider'); + expect(deleteSpy).toHaveBeenCalledWith({ name: '@simplens/mock' }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('CHANNEL_ROUTING_UPDATED', { + channel: 'mock', + }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_DELETED', { + provider_id: 'mock-provider', + }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PLUGIN_UNINSTALLED', { + plugin_name: '@simplens/mock', + }); + }); + + it('should uninstall and remove the plugin when no providers depend on it', async () => { + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ name: '@simplens/mock' } as any); + vi.spyOn(Provider, 'find').mockReturnValue({ + select: () => ({ + lean: vi.fn().mockResolvedValue([]), + }), + } as any); + const uninstallSpy = vi.spyOn(pluginFs, 'uninstallNpmPackage').mockReturnValue(undefined); + const deleteSpy = vi.spyOn(Plugin, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); + + await PluginManagerService.uninstallPlugin('@simplens/mock'); + + expect(uninstallSpy).toHaveBeenCalledWith('@simplens/mock'); + expect(deleteSpy).toHaveBeenCalledWith({ name: '@simplens/mock' }); + }); + }); +}); diff --git a/tests/unit/plugins/plugin-validation.test.ts b/tests/unit/plugins/plugin-validation.test.ts new file mode 100644 index 0000000..af18f7c --- /dev/null +++ b/tests/unit/plugins/plugin-validation.test.ts @@ -0,0 +1,85 @@ +/** + * Unit Tests for SimpleNSProvider Validation and Rollback + */ + +import { describe, it, expect, vi } from 'vitest'; +import { validateSimpleNSProvider } from '../../../src/plugins/loader/plugin-fs.js'; + +describe('validateSimpleNSProvider', () => { + const validProvider = { + manifest: { + name: '@simplens/mock', + displayName: 'Mock Provider', + version: '1.0.0', + channel: 'mock', + description: 'A mock provider for testing', + requiredCredentials: [], + }, + send: vi.fn(), + initialize: vi.fn(), + healthCheck: vi.fn(), + shutdown: vi.fn(), + getNotificationSchema: vi.fn(), + getRecipientSchema: vi.fn(), + getContentSchema: vi.fn(), + getRateLimitConfig: vi.fn(), + }; + + it('should accept a valid SimpleNSProvider implementation', () => { + const validated = validateSimpleNSProvider(validProvider, '@simplens/mock'); + expect(validated).toBeDefined(); + expect(validated.manifest.name).toBe('@simplens/mock'); + }); + + it('should reject a non-object or null export', () => { + expect(() => validateSimpleNSProvider(null, 'bad-pkg')).toThrow( + "Package 'bad-pkg' is not a valid SimpleNS plugin: default export must be a Provider object or class." + ); + expect(() => validateSimpleNSProvider(123, 'bad-pkg')).toThrow( + "Package 'bad-pkg' is not a valid SimpleNS plugin: default export must be a Provider object or class." + ); + }); + + it('should reject when manifest is missing', () => { + const missingManifest = { ...validProvider, manifest: undefined }; + expect(() => validateSimpleNSProvider(missingManifest, 'missing-manifest')).toThrow( + "Package 'missing-manifest' is not a valid SimpleNS plugin: missing 'manifest' definition." + ); + }); + + it('should reject when manifest fails schema validation (e.g. missing channel)', () => { + const invalidManifest = { + ...validProvider, + manifest: { + name: 'test', + // channel missing + displayName: 'Test', + description: 'desc', + version: '1.0.0', + requiredCredentials: [], + }, + }; + expect(() => validateSimpleNSProvider(invalidManifest, 'invalid-manifest')).toThrow( + "has an invalid plugin manifest" + ); + }); + + it('should reject arbitrary packages missing required provider methods (e.g. lodash / express)', () => { + const arbitraryPackage = { + default: () => ({ someUtility: true }), + manifest: { + name: 'arbitrary', + displayName: 'Arbitrary', + version: '1.0.0', + channel: 'email', + description: 'Not a real provider', + requiredCredentials: [], + }, + // missing send, initialize, healthCheck, etc. + }; + + expect(() => validateSimpleNSProvider(arbitraryPackage, 'lodash')).toThrow( + "Package 'lodash' is not a valid SimpleNS plugin. Missing required provider methods: send, initialize, healthCheck, shutdown, getNotificationSchema, getRecipientSchema, getContentSchema, getRateLimitConfig" + ); + }); +}); diff --git a/tests/unit/plugins/provider-manager.service.test.ts b/tests/unit/plugins/provider-manager.service.test.ts new file mode 100644 index 0000000..dc8c7aa --- /dev/null +++ b/tests/unit/plugins/provider-manager.service.test.ts @@ -0,0 +1,379 @@ +/** + * Unit Tests for ProviderManagerService + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { ProviderManagerService } from '../../../src/plugins/services/provider-manager.service.js'; +import Provider from '../../../src/database/models/provider.models.js'; +import Plugin from '../../../src/database/models/plugin.models.js'; +import ChannelRouting from '../../../src/database/models/channel-routing.models.js'; +import * as keypairManager from '../../../src/plugins/crypto/keypair-manager.js'; +import * as pluginFs from '../../../src/plugins/loader/plugin-fs.js'; +import { PluginRegistry } from '../../../src/plugins/loader/registry.js'; +import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; +import { ChannelRoutingService } from '../../../src/plugins/services/channel-routing.service.js'; + +describe('ProviderManagerService', () => { + beforeEach(() => { + vi.restoreAllMocks(); + PluginRegistry.clear(); + vi.spyOn(PluginSyncService, 'publish').mockResolvedValue(undefined); + }); + + describe('createProvider', () => { + it('should throw error if provider ID already exists', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ id: 'gmail-prod' } as any); + + await expect( + ProviderManagerService.createProvider({ + id: 'gmail-prod', + plugin_name: '@simplens/nodemailer-gmail', + credentials: { EMAIL_USER: 'test@example.com' }, + }) + ).rejects.toThrow("Provider with ID 'gmail-prod' already exists."); + }); + + it('should throw error if plugin is not installed', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue(null); + vi.spyOn(Plugin, 'findOne').mockResolvedValue(null); + + await expect( + ProviderManagerService.createProvider({ + id: 'gmail-prod', + plugin_name: '@simplens/nodemailer-gmail', + credentials: { EMAIL_USER: 'test@example.com' }, + }) + ).rejects.toThrow("Plugin '@simplens/nodemailer-gmail' is not installed."); + }); + + it('should encrypt credentials, create provider in DB, and broadcast sync event', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue(null); + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ + name: '@simplens/mock', + manifest: { channel: 'mock' }, + } as any); + + vi.spyOn(keypairManager, 'encryptCredentials').mockResolvedValue({ + encrypted_data: 'enc123', + iv: 'iv123', + auth_tag: 'tag123', + encrypted_dek: 'dek123', + }); + + const mockCreated = { + _id: '507f1f77bcf86cd799439011', + id: 'mock-test', + plugin_name: '@simplens/mock', + channel: 'mock', + enabled: true, + credentials: { encrypted_data: 'enc123' }, + options: { priority: 5 }, + toObject: () => mockCreated, + }; + + vi.spyOn(Provider, 'create').mockResolvedValue(mockCreated as any); + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue(null); + const setRoutingSpy = vi.spyOn(ChannelRoutingService, 'setChannelRouting').mockResolvedValue({} as any); + + const result = await ProviderManagerService.createProvider({ + id: 'mock-test', + plugin_name: '@simplens/mock', + credentials: {}, + options: { priority: 5 }, + }); + + expect(keypairManager.encryptCredentials).toHaveBeenCalled(); + expect(Provider.create).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'mock-test', + plugin_name: '@simplens/mock', + channel: 'mock', + options: { priority: 5 }, + }) + ); + expect(setRoutingSpy).toHaveBeenCalledWith('mock', { + default_provider_id: 'mock-test', + fallback_provider_ids: [], + partitions: 6, + }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_UPSERTED', { + provider_id: 'mock-test', + }); + expect(result.credentials_configured).toBe(true); + }); + + it('should not overwrite ChannelRouting if one already exists for the channel', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue(null); + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ + name: '@simplens/mock', + manifest: { channel: 'mock' }, + } as any); + + vi.spyOn(keypairManager, 'encryptCredentials').mockResolvedValue({ + encrypted_data: 'enc123', + iv: 'iv123', + auth_tag: 'tag123', + encrypted_dek: 'dek123', + }); + + const mockCreated = { + _id: '507f1f77bcf86cd799439011', + id: 'mock-test-2', + plugin_name: '@simplens/mock', + channel: 'mock', + enabled: true, + credentials: { encrypted_data: 'enc123' }, + options: {}, + toObject: () => mockCreated, + }; + + vi.spyOn(Provider, 'create').mockResolvedValue(mockCreated as any); + vi.spyOn(ProviderManagerService, 'loadAndRegisterProvider').mockResolvedValue(undefined); + vi.spyOn(ChannelRouting, 'findOne').mockResolvedValue({ + channel: 'mock', + default_provider_id: 'existing-provider', + } as any); + const setRoutingSpy = vi.spyOn(ChannelRoutingService, 'setChannelRouting'); + + await ProviderManagerService.createProvider({ + id: 'mock-test-2', + plugin_name: '@simplens/mock', + credentials: {}, + }); + + expect(setRoutingSpy).not.toHaveBeenCalled(); + }); + }); + + describe('updateProvider', () => { + it('should promote an active fallback provider to default in routing when default provider is disabled', async () => { + const mockDoc = { + _id: '507f1f77bcf86cd799439011', + id: 'gmail-prod', + plugin_name: '@simplens/mock', + channel: 'email', + enabled: true, + save: vi.fn().mockResolvedValue(undefined), + toObject: () => mockDoc, + }; + + vi.spyOn(Provider, 'findOne').mockResolvedValue(mockDoc as any); + vi.spyOn(ChannelRouting, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + _id: 'routing-1', + channel: 'email', + default_provider_id: 'gmail-prod', + fallback_provider_ids: ['smtp-backup', 'secondary-backup'], + }, + ]), + } as any); + + vi.spyOn(Provider, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { id: 'smtp-backup', channel: 'email', enabled: true }, + ]), + } as any); + + const updateRoutingSpy = vi.spyOn(ChannelRouting, 'updateOne').mockResolvedValue({ acknowledged: true } as any); + const setChannelConfigSpy = vi.spyOn(PluginRegistry, 'setChannelConfig'); + vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); + + await ProviderManagerService.updateProvider('gmail-prod', { enabled: false }); + + expect(mockDoc.enabled).toBe(false); + expect(mockDoc.save).toHaveBeenCalled(); + expect(PluginRegistry.unregister).toHaveBeenCalledWith('gmail-prod'); + expect(updateRoutingSpy).toHaveBeenCalledWith( + { _id: 'routing-1' }, + { + $set: { + default_provider_id: 'smtp-backup', + fallback_provider_ids: ['secondary-backup'], + }, + } + ); + expect(setChannelConfigSpy).toHaveBeenCalledWith('email', { + default: 'smtp-backup', + fallback: ['secondary-backup'], + }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('CHANNEL_ROUTING_UPDATED', { + channel: 'email', + }); + }); + }); + + describe('deleteProvider', () => { + it('should remove the provider from routing and promote the first fallback', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ id: 'gmail-prod' } as any); + vi.spyOn(ChannelRouting, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + channel: 'email', + default_provider_id: 'gmail-prod', + fallback_provider_ids: ['smtp-prod', 'backup-prod'], + }, + ]), + } as any); + vi.spyOn(Provider, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { id: 'smtp-prod', channel: 'email', enabled: true }, + { id: 'backup-prod', channel: 'email', enabled: true }, + ]), + } as any); + const updateSpy = vi.spyOn(ChannelRouting, 'updateOne').mockResolvedValue({ acknowledged: true } as any); + const deleteRoutingSpy = vi.spyOn(ChannelRouting, 'deleteOne').mockResolvedValue({ deletedCount: 0 } as any); + const setChannelConfigSpy = vi.spyOn(PluginRegistry, 'setChannelConfig'); + vi.spyOn(Provider, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); + vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); + + await ProviderManagerService.deleteProvider('gmail-prod'); + + expect(updateSpy).toHaveBeenCalledWith( + { channel: 'email' }, + { + default_provider_id: 'smtp-prod', + fallback_provider_ids: ['backup-prod'], + } + ); + expect(setChannelConfigSpy).toHaveBeenCalledWith('email', { + default: 'smtp-prod', + fallback: ['backup-prod'], + }); + expect(deleteRoutingSpy).not.toHaveBeenCalled(); + expect(PluginSyncService.publish).toHaveBeenCalledWith('CHANNEL_ROUTING_UPDATED', { + channel: 'email', + }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_DELETED', { + provider_id: 'gmail-prod', + }); + }); + + it('should retain default provider and only update fallbacks when deleting a fallback provider', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ id: 'smtp-prod' } as any); + vi.spyOn(ChannelRouting, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + channel: 'email', + default_provider_id: 'gmail-prod', + fallback_provider_ids: ['smtp-prod', 'backup-prod'], + }, + ]), + } as any); + vi.spyOn(Provider, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { id: 'gmail-prod', channel: 'email', enabled: true }, + { id: 'backup-prod', channel: 'email', enabled: true }, + ]), + } as any); + const updateSpy = vi.spyOn(ChannelRouting, 'updateOne').mockResolvedValue({ acknowledged: true } as any); + const deleteRoutingSpy = vi.spyOn(ChannelRouting, 'deleteOne').mockResolvedValue({ deletedCount: 0 } as any); + const setChannelConfigSpy = vi.spyOn(PluginRegistry, 'setChannelConfig'); + vi.spyOn(Provider, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); + vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); + + await ProviderManagerService.deleteProvider('smtp-prod'); + + expect(updateSpy).toHaveBeenCalledWith( + { channel: 'email' }, + { + fallback_provider_ids: ['backup-prod'], + } + ); + expect(setChannelConfigSpy).toHaveBeenCalledWith('email', { + default: 'gmail-prod', + fallback: ['backup-prod'], + }); + expect(deleteRoutingSpy).not.toHaveBeenCalled(); + expect(PluginSyncService.publish).toHaveBeenCalledWith('CHANNEL_ROUTING_UPDATED', { + channel: 'email', + }); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_DELETED', { + provider_id: 'smtp-prod', + }); + }); + + it('should delete the routing when the provider has no fallback replacement', async () => { + vi.spyOn(Provider, 'findOne').mockResolvedValue({ id: 'mock-unused' } as any); + vi.spyOn(ChannelRouting, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + channel: 'mock', + default_provider_id: 'mock-unused', + fallback_provider_ids: [], + }, + ]), + } as any); + vi.spyOn(Provider, 'find').mockReturnValue({ + lean: vi.fn().mockResolvedValue([]), + } as any); + const deleteRoutingSpy = vi.spyOn(ChannelRouting, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); + const deleteSpy = vi.spyOn(Provider, 'deleteOne').mockResolvedValue({ deletedCount: 1 } as any); + const unregisterSpy = vi.spyOn(PluginRegistry, 'unregister').mockReturnValue(true); + const setChannelConfigSpy = vi.spyOn(PluginRegistry, 'setChannelConfig'); + + await ProviderManagerService.deleteProvider('mock-unused'); + + expect(deleteRoutingSpy).toHaveBeenCalledWith({ channel: 'mock' }); + expect(setChannelConfigSpy).toHaveBeenCalledWith('mock', { + default: '', + fallback: [], + }); + expect(deleteSpy).toHaveBeenCalledWith({ id: 'mock-unused' }); + expect(unregisterSpy).toHaveBeenCalledWith('mock-unused'); + expect(PluginSyncService.publish).toHaveBeenCalledWith('PROVIDER_DELETED', { + provider_id: 'mock-unused', + }); + }); + }); + + describe('testProviderConnection', () => { + it('should return success true when health check passes', async () => { + const mockProviderInstance = { + initialize: vi.fn().mockResolvedValue(undefined), + healthCheck: vi.fn().mockResolvedValue(true), + }; + + vi.spyOn(pluginFs, 'importAndInstantiateProvider').mockResolvedValue(mockProviderInstance as any); + + const result = await ProviderManagerService.testProviderConnection({ + plugin_name: '@simplens/mock', + credentials: {}, + }); + + expect(result.success).toBe(true); + expect(mockProviderInstance.initialize).toHaveBeenCalled(); + expect(mockProviderInstance.healthCheck).toHaveBeenCalled(); + }); + + it('should decrypt stored credentials once and reuse them for blank-credential tests', async () => { + const mockProviderInstance = { + initialize: vi.fn().mockResolvedValue(undefined), + healthCheck: vi.fn().mockResolvedValue(true), + }; + const decryptSpy = vi.spyOn(keypairManager, 'decryptCredentials').mockResolvedValue({ + EMAIL_USER: 'stored-user', + EMAIL_PASS: 'stored-pass', + }); + + vi.spyOn(Provider, 'findOne').mockResolvedValue({ + id: 'cached-provider-test', + plugin_name: '@simplens/mock', + options: { priority: 2 }, + credentials: { encrypted_data: 'encrypted' }, + } as any); + vi.spyOn(pluginFs, 'importAndInstantiateProvider').mockResolvedValue(mockProviderInstance as any); + + await ProviderManagerService.testProviderConnection({ provider_id: 'cached-provider-test' }); + await ProviderManagerService.testProviderConnection({ provider_id: 'cached-provider-test' }); + + expect(decryptSpy).toHaveBeenCalledTimes(1); + expect(mockProviderInstance.initialize).toHaveBeenNthCalledWith(1, expect.objectContaining({ + credentials: { EMAIL_USER: 'stored-user', EMAIL_PASS: 'stored-pass' }, + })); + expect(mockProviderInstance.initialize).toHaveBeenNthCalledWith(2, expect.objectContaining({ + credentials: { EMAIL_USER: 'stored-user', EMAIL_PASS: 'stored-pass' }, + })); + }); + }); +}); diff --git a/tests/unit/plugins/registry.test.ts b/tests/unit/plugins/registry.test.ts index 8c3c3d6..ff621d8 100644 --- a/tests/unit/plugins/registry.test.ts +++ b/tests/unit/plugins/registry.test.ts @@ -92,6 +92,32 @@ describe('Plugin Registry', () => { }); }); + describe('registerOrReplace', () => { + it('should register a new provider if not present', () => { + const gmail = createMockProvider({ name: 'gmail', channel: 'email' }); + PluginRegistry.registerOrReplace(gmail as any, 'gmail', 1); + + expect(PluginRegistry.has('gmail')).toBe(true); + expect(PluginRegistry.get('gmail')).toBeDefined(); + }); + + it('should replace an existing provider and preserve channelConfig default and fallback', () => { + const gmail1 = createMockProvider({ name: 'gmail-v1', channel: 'email' }); + const gmail2 = createMockProvider({ name: 'gmail-v2', channel: 'email' }); + const smtp = createMockProvider({ name: 'smtp', channel: 'email' }); + + PluginRegistry.register(gmail1 as any, 'gmail', 1); + PluginRegistry.register(smtp as any, 'smtp', 2); + PluginRegistry.setChannelConfig('email', { default: 'gmail', fallback: ['smtp'] }); + + PluginRegistry.registerOrReplace(gmail2 as any, 'gmail', 5); + + expect(PluginRegistry.get('gmail')?.manifest.name).toBe('gmail-v2'); + expect(PluginRegistry.getDefaultProviderId('email')).toBe('gmail'); + expect(PluginRegistry.getFallbackProviderIds('email')).toEqual(['smtp']); + }); + }); + describe('get', () => { it('should return provider by ID', () => { const mockProvider = createMockProvider({ name: 'gmail', channel: 'email' }); @@ -154,6 +180,18 @@ describe('Plugin Registry', () => { const provider = PluginRegistry.getDefaultProvider('unknown'); expect(provider).toBeUndefined(); }); + + it('should fall back to registered fallback provider when configured default is unregistered', () => { + const fallback = createMockProvider({ name: 'fallback', channel: 'email' }); + PluginRegistry.register(fallback as any, 'fallback', 1); + + // Configure default to an unregistered provider, with fallback configured + PluginRegistry.setChannelConfig('email', { default: 'unregistered-default', fallback: ['fallback'] }); + + const defaultProvider = PluginRegistry.getDefaultProvider('email'); + expect(defaultProvider).toBeDefined(); + expect(defaultProvider?.manifest.name).toBe('fallback'); + }); }); describe('getDefaultProviderId', () => { @@ -167,6 +205,16 @@ describe('Plugin Registry', () => { expect(defaultId).toBe('gmail'); }); + it('should return fallback provider ID when configured default is unregistered', () => { + const fallback = createMockProvider({ name: 'fallback', channel: 'email' }); + PluginRegistry.register(fallback as any, 'fallback', 1); + + PluginRegistry.setChannelConfig('email', { default: 'unregistered-default', fallback: ['fallback'] }); + + const defaultId = PluginRegistry.getDefaultProviderId('email'); + expect(defaultId).toBe('fallback'); + }); + it('should return undefined for unknown channel', () => { const defaultId = PluginRegistry.getDefaultProviderId('unknown'); expect(defaultId).toBeUndefined(); diff --git a/tests/unit/plugins/router.test.ts b/tests/unit/plugins/router.test.ts index 7fdd848..e988ff8 100644 --- a/tests/unit/plugins/router.test.ts +++ b/tests/unit/plugins/router.test.ts @@ -22,9 +22,14 @@ vi.mock('../../../src/processors/unified/unified.logger.js', () => ({ warn: vi.fn(), error: vi.fn(), debug: vi.fn(), + success: vi.fn(), }, })); +vi.mock('../../../src/processors/shared/rate-limiter.js', () => ({ + consumeToken: vi.fn().mockResolvedValue({ allowed: true }), +})); + // Create a mock provider factory function createMockProvider(options: { name: string; @@ -71,6 +76,7 @@ describe('Provider Router', () => { let validateNotification: typeof import('../../../src/plugins/loader/router.js').validateNotification; let validateNotificationForProvider: typeof import('../../../src/plugins/loader/router.js').validateNotificationForProvider; let resolveFallbackProviderId: typeof import('../../../src/plugins/loader/router.js').resolveFallbackProviderId; + let getProviderCascade: typeof import('../../../src/plugins/loader/router.js').getProviderCascade; let getRateLimitConfig: typeof import('../../../src/plugins/loader/router.js').getRateLimitConfig; beforeEach(async () => { @@ -84,6 +90,7 @@ describe('Provider Router', () => { validateNotification = routerModule.validateNotification; validateNotificationForProvider = routerModule.validateNotificationForProvider; resolveFallbackProviderId = routerModule.resolveFallbackProviderId; + getProviderCascade = routerModule.getProviderCascade; getRateLimitConfig = routerModule.getRateLimitConfig; PluginRegistry.clear(); @@ -158,6 +165,41 @@ describe('Provider Router', () => { expect(fallback.send).toHaveBeenCalled(); }); + it('should record execution attempts with timing, provider, and error details', async () => { + const primary = createMockProvider({ + name: 'primary', + channel: 'email', + sendResult: { + success: false, + error: { code: 'AUTH_FAILED', message: 'Invalid API key', retryable: false } + } + }); + const fallback = createMockProvider({ name: 'fallback', channel: 'email' }); + + PluginRegistry.register(primary as any, 'primary', 2); + PluginRegistry.register(fallback as any, 'fallback', 1); + PluginRegistry.setChannelConfig('email', { default: 'primary', fallback: 'fallback' }); + + const result = await sendWithFallback('email', createNotification('email')); + + expect(result.success).toBe(true); + expect(result.provider).toBe('fallback'); + expect(result.attempts).toHaveLength(2); + expect(result.attempts?.[0]).toMatchObject({ + provider: 'primary', + status: 'failed', + error_code: 'AUTH_FAILED', + error_message: 'Invalid API key', + retryable: false, + }); + expect(result.attempts?.[0].duration_ms).toBeDefined(); + expect(result.attempts?.[1]).toMatchObject({ + provider: 'fallback', + status: 'delivered', + }); + expect(result.attempts?.[1].duration_ms).toBeDefined(); + }); + it('should NOT try fallback on retryable error', async () => { const primary = createMockProvider({ name: 'primary', @@ -377,6 +419,35 @@ describe('Provider Router', () => { }); }); + describe('getProviderCascade', () => { + it('should return full cascade of primary and fallback providers', () => { + const primary = createMockProvider({ name: 'primary', channel: 'email' }); + const fb1 = createMockProvider({ name: 'fb1', channel: 'email' }); + const fb2 = createMockProvider({ name: 'fb2', channel: 'email' }); + + PluginRegistry.register(primary as any, 'primary', 3); + PluginRegistry.register(fb1 as any, 'fb1', 2); + PluginRegistry.register(fb2 as any, 'fb2', 1); + PluginRegistry.setChannelConfig('email', { default: 'primary', fallback: ['fb1', 'fb2'] }); + + expect(getProviderCascade('email')).toEqual(['primary', 'fb1', 'fb2']); + }); + + it('should return cascade starting from currentProviderId onwards', () => { + const primary = createMockProvider({ name: 'primary', channel: 'email' }); + const fb1 = createMockProvider({ name: 'fb1', channel: 'email' }); + const fb2 = createMockProvider({ name: 'fb2', channel: 'email' }); + + PluginRegistry.register(primary as any, 'primary', 3); + PluginRegistry.register(fb1 as any, 'fb1', 2); + PluginRegistry.register(fb2 as any, 'fb2', 1); + PluginRegistry.setChannelConfig('email', { default: 'primary', fallback: ['fb1', 'fb2'] }); + + expect(getProviderCascade('email', 'fb1')).toEqual(['fb1', 'fb2']); + expect(getProviderCascade('email', 'fb2')).toEqual(['fb2']); + }); + }); + describe('getRateLimitConfig', () => { it('should return rate limit config for provider', () => { const gmail = createMockProvider({ diff --git a/tests/unit/plugins/yaml-migrator.test.ts b/tests/unit/plugins/yaml-migrator.test.ts new file mode 100644 index 0000000..4b7dc7b --- /dev/null +++ b/tests/unit/plugins/yaml-migrator.test.ts @@ -0,0 +1,282 @@ +/** + * Unit Tests for YamlMigrator + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import * as fs from 'fs'; +import { YamlMigrator } from '../../../src/plugins/bootstrap/yaml-migrator.js'; +import Plugin from '../../../src/database/models/plugin.models.js'; +import Provider from '../../../src/database/models/provider.models.js'; +import ChannelRouting from '../../../src/database/models/channel-routing.models.js'; +import * as pluginFs from '../../../src/plugins/loader/plugin-fs.js'; +import * as keypairManager from '../../../src/plugins/crypto/keypair-manager.js'; +import { PluginSyncService } from '../../../src/plugins/sync/plugin-sync.service.js'; + +vi.mock('fs', async () => { + const actual = await vi.importActual('fs'); + return { + ...actual, + existsSync: vi.fn(), + readFileSync: vi.fn(), + }; +}); + +describe('YamlMigrator Unit Tests', () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.spyOn(PluginSyncService, 'publish').mockResolvedValue(undefined); + vi.spyOn(keypairManager, 'encryptCredentials').mockResolvedValue({ + encrypted_data: 'enc-data', + iv: 'enc-iv', + auth_tag: 'enc-tag', + encrypted_dek: 'enc-dek', + }); + }); + + it('should skip migration if providers already exist in MongoDB', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(2 as any); + const findConfigSpy = vi.spyOn(pluginFs, 'findConfigFile'); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(false); + expect(findConfigSpy).not.toHaveBeenCalled(); + }); + + it('should return false if no config file is found', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue(null); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(false); + }); + + it('should return false if config has no providers', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue('./simplens.config.yaml'); + vi.mocked(fs.readFileSync).mockReturnValue('channels: {}'); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(false); + }); + + it('should skip provider creation and not create generic provider if npm install fails', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue('./simplens.config.yaml'); + vi.mocked(fs.existsSync).mockReturnValue(false); + vi.mocked(fs.readFileSync).mockReturnValue(` +providers: + - id: broken-provider + package: '@simplens/broken-pkg' + credentials: + apiKey: test-key +`); + + vi.spyOn(pluginFs, 'installNpmPackage').mockImplementation(() => { + throw new Error('NPM install 404 Not Found'); + }); + const extractManifestSpy = vi.spyOn(pluginFs, 'extractPackageManifest'); + const providerUpsertSpy = vi.spyOn(Provider, 'findOneAndUpdate'); + const pluginUpsertSpy = vi.spyOn(Plugin, 'findOneAndUpdate'); + vi.spyOn(Plugin, 'findOne').mockResolvedValue(null); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + // Migration returns false because 0 providers were migrated + expect(result).toBe(false); + // Manifest extraction shouldn't be called if install failed + expect(extractManifestSpy).not.toHaveBeenCalled(); + // No Plugin record created + expect(pluginUpsertSpy).not.toHaveBeenCalled(); + // CRUCIAL: No Provider record created, generic provider avoided + expect(providerUpsertSpy).not.toHaveBeenCalled(); + }); + + it('should skip provider creation if manifest extraction fails', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue('./simplens.config.yaml'); + vi.mocked(fs.existsSync).mockReturnValue(false); + vi.mocked(fs.readFileSync).mockReturnValue(` +providers: + - id: broken-manifest-provider + package: '@simplens/corrupted-pkg' + credentials: + apiKey: test-key +`); + + vi.spyOn(pluginFs, 'installNpmPackage').mockReturnValue(undefined); + vi.spyOn(pluginFs, 'extractPackageManifest').mockRejectedValue(new Error('Invalid package manifest')); + const providerUpsertSpy = vi.spyOn(Provider, 'findOneAndUpdate'); + const pluginUpsertSpy = vi.spyOn(Plugin, 'findOneAndUpdate'); + vi.spyOn(Plugin, 'findOne').mockResolvedValue(null); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(false); + expect(pluginUpsertSpy).not.toHaveBeenCalled(); + expect(providerUpsertSpy).not.toHaveBeenCalled(); + }); + + it('should migrate valid provider and omit failed provider, promoting fallback if default failed', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue('./simplens.config.yaml'); + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.readFileSync).mockReturnValue(` +providers: + - id: default-email + package: '@simplens/broken-email' + credentials: + key: val + - id: fallback-email + package: '@simplens/good-email' + credentials: + key: val +channels: + email: + default: default-email + fallback: + - fallback-email +`); + + // First package fails manifest, second succeeds + vi.spyOn(pluginFs, 'extractPackageManifest').mockImplementation(async (pkg: string) => { + if (pkg === '@simplens/broken-email') { + throw new Error('Broken manifest'); + } + return { + manifest: { + name: '@simplens/good-email', + displayName: 'Good Email Provider', + version: '1.0.0', + channel: 'email', + description: 'Good email provider', + requiredCredentials: ['key'], + }, + version: '1.0.0', + }; + }); + + const pluginUpsertSpy = vi.spyOn(Plugin, 'findOneAndUpdate').mockResolvedValue({} as any); + const providerUpsertSpy = vi.spyOn(Provider, 'findOneAndUpdate').mockResolvedValue({} as any); + const routingUpsertSpy = vi.spyOn(ChannelRouting, 'findOneAndUpdate').mockResolvedValue({} as any); + + vi.spyOn(Plugin, 'findOne').mockImplementation(async (query: any) => { + if (query.name === '@simplens/good-email') { + return { + name: '@simplens/good-email', + status: 'installed', + manifest: { channel: 'email' }, + } as any; + } + return null; + }); + + vi.spyOn(Provider, 'findOne').mockImplementation(async (query: any) => { + if (query.id === 'fallback-email' && query.channel === 'email') { + return { id: 'fallback-email', channel: 'email' } as any; + } + return null; + }); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(true); + + // Only good-email plugin is saved + expect(pluginUpsertSpy).toHaveBeenCalledTimes(1); + expect(pluginUpsertSpy).toHaveBeenCalledWith( + { name: '@simplens/good-email' }, + expect.objectContaining({ name: '@simplens/good-email', status: 'installed' }), + expect.any(Object) + ); + + // Only fallback-email provider is created (with channel 'email') + expect(providerUpsertSpy).toHaveBeenCalledTimes(1); + expect(providerUpsertSpy).toHaveBeenCalledWith( + { id: 'fallback-email' }, + expect.objectContaining({ + id: 'fallback-email', + plugin_name: '@simplens/good-email', + channel: 'email', + enabled: true, + }), + expect.any(Object) + ); + + // Channel routing promoted fallback-email to default since default-email failed + expect(routingUpsertSpy).toHaveBeenCalledWith( + { channel: 'email' }, + expect.objectContaining({ + channel: 'email', + default_provider_id: 'fallback-email', + fallback_provider_ids: [], + }), + expect.any(Object) + ); + }); + + it('should migrate all providers and channel routings successfully when all succeed', async () => { + vi.spyOn(Provider, 'countDocuments').mockResolvedValue(0 as any); + vi.spyOn(pluginFs, 'findConfigFile').mockReturnValue('./simplens.config.yaml'); + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.readFileSync).mockReturnValue(` +providers: + - id: resend-provider + package: '@simplens/resend' + credentials: + apiKey: resend-123 +channels: + email: + default: resend-provider +`); + + vi.spyOn(pluginFs, 'extractPackageManifest').mockResolvedValue({ + manifest: { + name: '@simplens/resend', + displayName: 'Resend Provider', + version: '1.2.0', + channel: 'email', + description: 'Resend email provider', + requiredCredentials: ['apiKey'], + }, + version: '1.2.0', + }); + + vi.spyOn(Plugin, 'findOneAndUpdate').mockResolvedValue({} as any); + vi.spyOn(Provider, 'findOneAndUpdate').mockResolvedValue({} as any); + vi.spyOn(ChannelRouting, 'findOneAndUpdate').mockResolvedValue({} as any); + + vi.spyOn(Plugin, 'findOne').mockResolvedValue({ + name: '@simplens/resend', + status: 'installed', + manifest: { channel: 'email' }, + } as any); + + const publishSpy = vi.spyOn(PluginSyncService, 'publish'); + + const result = await YamlMigrator.migrateYamlIfNeeded(); + + expect(result).toBe(true); + expect(Provider.findOneAndUpdate).toHaveBeenCalledWith( + { id: 'resend-provider' }, + expect.objectContaining({ + id: 'resend-provider', + plugin_name: '@simplens/resend', + channel: 'email', + enabled: true, + }), + expect.any(Object) + ); + expect(ChannelRouting.findOneAndUpdate).toHaveBeenCalledWith( + { channel: 'email' }, + expect.objectContaining({ + channel: 'email', + default_provider_id: 'resend-provider', + }), + expect.any(Object) + ); + expect(publishSpy).toHaveBeenCalledWith('SYSTEM_RELOAD', {}); + }); +}); diff --git a/tests/unit/processors/delayed-consumer-health.test.ts b/tests/unit/processors/delayed-consumer-health.test.ts new file mode 100644 index 0000000..4284dd4 --- /dev/null +++ b/tests/unit/processors/delayed-consumer-health.test.ts @@ -0,0 +1,92 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const eventListeners: Record void)[]> = {}; + +const kafkaConsumerMock = { + connect: vi.fn().mockResolvedValue(undefined), + subscribe: vi.fn().mockResolvedValue(undefined), + run: vi.fn().mockResolvedValue(undefined), + stop: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + commitOffsets: vi.fn().mockResolvedValue(undefined), + on: vi.fn().mockImplementation((event: string, listener: (e: unknown) => void) => { + if (!eventListeners[event]) { + eventListeners[event] = []; + } + eventListeners[event].push(listener); + }), + events: { + CRASH: 'consumer.crash', + HEARTBEAT: 'consumer.heartbeat', + STOP: 'consumer.stop', + DISCONNECT: 'consumer.disconnect', + CONNECT: 'consumer.connect', + }, +}; + +vi.mock('@src/config/kafka.config.js', () => ({ + kafka: { + consumer: vi.fn(() => kafkaConsumerMock), + }, +})); + +vi.mock('@src/processors/delayed/delayed.queue.js', () => ({ + addToDelayedQueue: vi.fn().mockResolvedValue(undefined), +})); + +describe('Delayed Consumer Health Checks', () => { + beforeEach(() => { + vi.clearAllMocks(); + for (const key of Object.keys(eventListeners)) { + delete eventListeners[key]; + } + }); + + it('should report unhealthy when consumer is not started', async () => { + const { isDelayedConsumerHealthy } = await import('@src/processors/delayed/delayed.consumer.js'); + expect(isDelayedConsumerHealthy()).toBe(false); + }); + + it('should report healthy once consumer starts', async () => { + const { startDelayedConsumer, isDelayedConsumerHealthy, stopDelayedConsumer } = await import( + '@src/processors/delayed/delayed.consumer.js' + ); + + await startDelayedConsumer(); + expect(isDelayedConsumerHealthy()).toBe(true); + + await stopDelayedConsumer(); + expect(isDelayedConsumerHealthy()).toBe(false); + }); + + it('should report unhealthy if consumer crashes', async () => { + const { startDelayedConsumer, isDelayedConsumerHealthy, stopDelayedConsumer } = await import( + '@src/processors/delayed/delayed.consumer.js' + ); + + await startDelayedConsumer(); + expect(isDelayedConsumerHealthy()).toBe(true); + + // Emit crash event + const crashHandlers = eventListeners['consumer.crash'] || []; + for (const handler of crashHandlers) { + handler({ payload: { error: new Error('Broker disconnected') } }); + } + + expect(isDelayedConsumerHealthy()).toBe(false); + + await stopDelayedConsumer(); + }); + + it('should report unhealthy if heartbeat is stale', async () => { + const { startDelayedConsumer, isDelayedConsumerHealthy, stopDelayedConsumer } = await import( + '@src/processors/delayed/delayed.consumer.js' + ); + + await startDelayedConsumer(); + // With negative allowance, immediately considered stale + expect(isDelayedConsumerHealthy(-1)).toBe(false); + + await stopDelayedConsumer(); + }); +}); diff --git a/tests/unit/processors/unified.consumer.test.ts b/tests/unit/processors/unified.consumer.test.ts index 77d591b..9f9c3e5 100644 --- a/tests/unit/processors/unified.consumer.test.ts +++ b/tests/unit/processors/unified.consumer.test.ts @@ -10,6 +10,7 @@ const publishDelayedMock = vi.fn().mockResolvedValue(undefined); const publishStatusMock = vi.fn().mockResolvedValue(undefined); const consumeTokenMock = vi.fn(); const resolveFallbackProviderIdMock = vi.fn(); +const getProviderCascadeMock = vi.fn(); const validateNotificationForProviderMock = vi.fn(); const sendWithFallbackMock = vi.fn(); @@ -36,12 +37,21 @@ const kafkaConsumerMock = { commitOffsets: commitOffsetsMock, stop: vi.fn().mockResolvedValue(undefined), disconnect: vi.fn().mockResolvedValue(undefined), + on: vi.fn(), + events: { + CRASH: 'consumer.crash', + HEARTBEAT: 'consumer.heartbeat', + STOP: 'consumer.stop', + DISCONNECT: 'consumer.disconnect', + CONNECT: 'consumer.connect', + }, }; vi.mock('../../../src/config/kafka.config.js', () => ({ kafka: { consumer: vi.fn(() => kafkaConsumerMock), }, + ensureChannelTopic: vi.fn().mockResolvedValue(undefined), })); vi.mock('../../../src/plugins/index.js', () => ({ @@ -50,6 +60,7 @@ vi.mock('../../../src/plugins/index.js', () => ({ getDefaultProviderId: pluginRegistryGetDefaultProviderIdMock, }, sendWithFallback: sendWithFallbackMock, + getProviderCascade: getProviderCascadeMock, resolveFallbackProviderId: resolveFallbackProviderIdMock, validateNotificationForProvider: validateNotificationForProviderMock, })); @@ -126,6 +137,9 @@ describe('scheduleFallbackProviderHandoff', () => { }); tryAcquireProcessingLockMock.mockResolvedValue({ canProcess: true, isRetry: false }); consumeTokenMock.mockResolvedValue({ allowed: true }); + getProviderCascadeMock.mockImplementation((_channel: string, currentProviderId?: string) => { + return currentProviderId ? [currentProviderId] : ['primary']; + }); sendWithFallbackMock.mockResolvedValue({ success: true, messageId: 'msg-1' }); buildDelayedPayloadGenericMock.mockReturnValue({ @@ -278,6 +292,21 @@ describe('processMessage via consumer run loop', () => { await stopUnifiedConsumer('email'); }; + beforeEach(() => { + vi.clearAllMocks(); + sendWithFallbackMock.mockResolvedValue({ success: true, messageId: 'msg-1' }); + consumeTokenMock.mockResolvedValue({ allowed: true }); + getProviderCascadeMock.mockImplementation((_channel: string, currentProviderId?: string) => { + return currentProviderId ? [currentProviderId] : ['primary']; + }); + resolveFallbackProviderIdMock.mockReturnValue(undefined); + tryAcquireProcessingLockMock.mockResolvedValue({ canProcess: true, isRetry: false }); + providerSafeParseMock.mockImplementation((notification) => ({ + success: true, + data: notification, + })); + }); + it('should set failed, publish failure status, and alert for non-retryable provider errors', async () => { sendWithFallbackMock.mockResolvedValue({ success: false, @@ -304,34 +333,64 @@ describe('processMessage via consumer run loop', () => { expect(commitOffsetsMock).toHaveBeenCalledTimes(1); }); - it('should hand off to fallback when rate-limit retry budget is exceeded', async () => { - consumeTokenMock.mockResolvedValue({ allowed: false, retryAfterMs: 1000 }); - resolveFallbackProviderIdMock.mockReturnValue('fallback'); - validateNotificationForProviderMock.mockImplementation((_providerId, notification) => ({ - success: true, - data: notification, + it('should cascade to fallback provider in real time when primary is rate-limited and deliver immediately', async () => { + getProviderCascadeMock.mockReturnValue(['primary', 'fallback']); + consumeTokenMock.mockImplementation(async (providerId) => { + if (providerId === 'primary') { + return { allowed: false, retryAfterMs: 5000 }; + } + return { allowed: true, remainingTokens: 10 }; + }); + + await startAndProcess(createNotification()); + + expect(consumeTokenMock).toHaveBeenCalledWith('primary'); + expect(consumeTokenMock).toHaveBeenCalledWith('fallback'); + expect(sendWithFallbackMock).toHaveBeenCalledWith( + 'email', + expect.objectContaining({ provider: 'fallback' }) + ); + expect(setDeliveredMock).toHaveBeenCalledWith('notif-123', 0); + expect(publishStatusMock).toHaveBeenCalledWith(expect.objectContaining({ + notification_id: 'notif-123', + status: 'delivered', + provider: 'fallback', + provider_history: expect.arrayContaining([ + expect.objectContaining({ + provider: 'primary', + status: 'rate_limited', + error_code: 'RATE_LIMITED' + }) + ]) })); + expect(commitOffsetsMock).toHaveBeenCalledTimes(1); + }); - await startAndProcess(createNotification({ retry_count: Number.MAX_SAFE_INTEGER })); + it('should push to delayed queue with fixed delay when all cascade providers are rate limited', async () => { + getProviderCascadeMock.mockReturnValue(['primary', 'fallback']); + consumeTokenMock.mockResolvedValue({ allowed: false, retryAfterMs: 5000 }); - expect(resolveFallbackProviderIdMock).toHaveBeenCalledWith('email', 'primary'); + await startAndProcess(createNotification({ retry_count: 0 })); + + expect(consumeTokenMock).toHaveBeenCalledWith('primary'); + expect(consumeTokenMock).toHaveBeenCalledWith('fallback'); + expect(setRateLimitedMock).toHaveBeenCalledWith('notif-123', 0); expect(buildDelayedPayloadGenericMock).toHaveBeenCalledWith( - expect.objectContaining({ provider: 'fallback' }), + expect.anything(), 'email', - 0 + 1, + 5000 ); expect(publishDelayedMock).toHaveBeenCalledTimes(1); - expect(publishStatusMock).not.toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); expect(commitOffsetsMock).toHaveBeenCalledTimes(1); }); - it('should set failed when retry budget exceeded and current provider is fallback', async () => { - consumeTokenMock.mockResolvedValue({ allowed: false, retryAfterMs: 1000 }); - resolveFallbackProviderIdMock.mockReturnValue(undefined); + it('should set failed when retry budget exceeded and all cascade providers are rate-limited', async () => { + getProviderCascadeMock.mockReturnValue(['primary', 'fallback']); + consumeTokenMock.mockResolvedValue({ allowed: false, retryAfterMs: 5000 }); - await startAndProcess(createNotification({ provider: 'fallback', retry_count: Number.MAX_SAFE_INTEGER })); + await startAndProcess(createNotification({ retry_count: Number.MAX_SAFE_INTEGER })); - expect(resolveFallbackProviderIdMock).toHaveBeenCalledWith('email', 'fallback'); expect(setFailedMock).toHaveBeenCalledWith('notif-123', Number.MAX_SAFE_INTEGER); expect(publishStatusMock).toHaveBeenCalledWith(expect.objectContaining({ notification_id: 'notif-123', @@ -347,3 +406,66 @@ describe('processMessage via consumer run loop', () => { expect(commitOffsetsMock).toHaveBeenCalledTimes(1); }); }); + +describe('areUnifiedConsumersHealthy', () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + }); + + it('should report healthy when in standby mode with no expected channels', async () => { + const { areUnifiedConsumersHealthy } = await import('../../../src/processors/unified/unified.consumer.js'); + const result = areUnifiedConsumersHealthy([]); + expect(result.healthy).toBe(true); + expect(result.details).toEqual({}); + }); + + it('should report unhealthy if expected channel has no consumer running', async () => { + const { areUnifiedConsumersHealthy } = await import('../../../src/processors/unified/unified.consumer.js'); + const result = areUnifiedConsumersHealthy(['email']); + expect(result.healthy).toBe(false); + expect(result.details.email.running).toBe(false); + expect(result.details.email.error).toContain('not initialized or missing'); + }); + + it('should report healthy once consumer starts and receives heartbeat', async () => { + const { startUnifiedConsumer, areUnifiedConsumersHealthy, stopAllConsumers } = await import('../../../src/processors/unified/unified.consumer.js'); + await startUnifiedConsumer('email'); + + const result = areUnifiedConsumersHealthy(['email']); + expect(result.healthy).toBe(true); + expect(result.details.email.running).toBe(true); + expect(result.details.email.crashed).toBe(false); + + await stopAllConsumers(); + }); + + it('should report unhealthy if consumer has crashed', async () => { + let crashListener: ((event: unknown) => void) | undefined; + kafkaConsumerMock.on.mockImplementation((event: string, listener: (e: unknown) => void) => { + if (event === 'consumer.crash') { + crashListener = listener; + } + }); + + const { startUnifiedConsumer, areUnifiedConsumersHealthy, stopAllConsumers } = await import('../../../src/processors/unified/unified.consumer.js'); + await startUnifiedConsumer('email'); + + // Simulate Kafka consumer crash event + if (crashListener) { + crashListener({ + payload: { + error: new Error('Kafka broker connection lost'), + restart: false, + } + }); + } + + const result = areUnifiedConsumersHealthy(['email']); + expect(result.healthy).toBe(false); + expect(result.details.email.crashed).toBe(true); + expect(result.details.email.error).toContain('Kafka broker connection lost'); + + await stopAllConsumers(); + }); +}); diff --git a/tests/unit/shared/rate-limiter.test.ts b/tests/unit/shared/rate-limiter.test.ts index 19a99ab..3ca39d1 100644 --- a/tests/unit/shared/rate-limiter.test.ts +++ b/tests/unit/shared/rate-limiter.test.ts @@ -1,8 +1,6 @@ /** * Unit Tests for Rate Limiter Module * Tests token bucket rate limiting with mocked Redis - * - * Updated for plugin-based architecture - uses dynamic channel strings. */ import { describe, it, expect, vi, beforeEach } from 'vitest'; @@ -21,6 +19,7 @@ vi.mock('../../../src/config/redis.config.js', () => ({ vi.mock('../../../src/config/env.config.js', () => ({ env: { + RATE_LIMIT_RETRY_DELAY_MS: 5000, EMAIL_RATE_LIMIT_TOKENS: 100, EMAIL_RATE_LIMIT_REFILL_RATE: 10, WHATSAPP_RATE_LIMIT_TOKENS: 50, @@ -28,6 +27,29 @@ vi.mock('../../../src/config/env.config.js', () => ({ }, })); +vi.mock('../../../src/database/models/provider.models.js', () => ({ + default: { + find: vi.fn().mockReturnValue({ + sort: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + id: 'email-provider', + channel: 'email', + plugin_name: 'nodemailer-gmail', + enabled: true, + }, + { + id: 'sms-provider', + channel: 'sms', + plugin_name: 'twilio-sms', + enabled: true, + }, + ]), + }), + }), + }, +})); + describe('Rate Limiter Module', () => { beforeEach(() => { vi.clearAllMocks(); @@ -37,8 +59,8 @@ describe('Rate Limiter Module', () => { it('should return allowed: true when tokens are available', async () => { const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - // Mock: Lua script returns [1, 99, 0, 0] meaning allowed with 99 remaining - mockRedisClient.eval.mockResolvedValue([1, 99, 0, 0]); + // Mock: Lua script returns [1, 99, 0] meaning allowed with 99 remaining + mockRedisClient.eval.mockResolvedValue([1, 99, 0]); const result = await consumeToken('email'); @@ -47,48 +69,36 @@ describe('Rate Limiter Module', () => { expect(result.retryAfterMs).toBeUndefined(); }); - it('should return allowed: false when rate limited', async () => { + it('should return allowed: false with fixed retry delay when rate limited', async () => { const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - // Mock: Lua script returns [0, 0, 100, 0] meaning denied, 0 tokens, retry in 100ms, queue position 0 - mockRedisClient.eval.mockResolvedValue([0, 0, 100, 0]); + // Mock: Lua script returns [0, 0, 5000] meaning denied, 0 tokens, retry in 5000ms + mockRedisClient.eval.mockResolvedValue([0, 0, 5000]); const result = await consumeToken('email'); expect(result.allowed).toBe(false); expect(result.remainingTokens).toBe(0); - expect(result.retryAfterMs).toBe(100); + expect(result.retryAfterMs).toBe(5000); }); - it('should use email config for email channel', async () => { + it('should use cluster-safe hash tag keys for Redis evaluation', async () => { const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.eval.mockResolvedValue([1, 99, 0, 0]); + mockRedisClient.eval.mockResolvedValue([1, 99, 0]); await consumeToken('email'); - // Verify the Lua script was called with email configuration - // eval(script, numKeys, key1, key2, arg1, arg2, arg3) - // arg1=maxTokens, arg2=refillRate, arg3=now expect(mockRedisClient.eval).toHaveBeenCalled(); const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) - expect(callArgs[5]).toBe('100'); // maxTokens for email (ARGV[1]) - expect(callArgs[6]).toBe('10'); // refillRate for email (ARGV[2]) - }); - - it('should use default config for whatsapp channel', async () => { - const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - - mockRedisClient.eval.mockResolvedValue([1, 99, 0, 0]); - - await consumeToken('whatsapp'); - - const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) - // Plugin system defaults all channels to same config now - expect(callArgs[5]).toBe('100'); // maxTokens (default) - expect(callArgs[6]).toBe('10'); // refillRate (default) + expect(callArgs[1]).toBe(4); // 4 keys passed + expect(callArgs[2]).toBe('ratelimit:{email}:tokens'); + expect(callArgs[3]).toBe('ratelimit:{email}:last_refill'); + expect(callArgs[4]).toBe('ratelimit:{email}:exhausted_count'); + expect(callArgs[5]).toBe('ratelimit:{email}:last_exhausted'); + expect(callArgs[6]).toBe('100'); // maxTokens + expect(callArgs[7]).toBe('10'); // refillRate + expect(callArgs[9]).toBe('5000'); // retry delay }); }); @@ -96,24 +106,21 @@ describe('Rate Limiter Module', () => { it('should return max tokens when no record exists', async () => { const { getTokenCount } = await import('../../../src/processors/shared/rate-limiter.js'); - // No existing keys mockRedisClient.mget.mockResolvedValue([null, null]); const count = await getTokenCount('email'); - expect(count).toBe(100); // EMAIL_RATE_LIMIT_TOKENS + expect(count).toBe(100); }); it('should return current token count from Redis', async () => { const { getTokenCount } = await import('../../../src/processors/shared/rate-limiter.js'); const now = Date.now(); - // Current tokens = 50, last refill = now (no time passed) mockRedisClient.mget.mockResolvedValue(['50', now.toString()]); const count = await getTokenCount('email'); - // Should be approximately 50 (may have slight variation due to timing) expect(count).toBeGreaterThanOrEqual(50); expect(count).toBeLessThanOrEqual(100); }); @@ -121,27 +128,91 @@ describe('Rate Limiter Module', () => { it('should account for token refill over time', async () => { const { getTokenCount } = await import('../../../src/processors/shared/rate-limiter.js'); - // 1 second ago with 50 tokens, refill rate of 10/sec const oneSecondAgo = Date.now() - 1000; mockRedisClient.mget.mockResolvedValue(['50', oneSecondAgo.toString()]); const count = await getTokenCount('email'); - // Should be 50 + 10 = 60 (approximately) expect(count).toBeGreaterThanOrEqual(59); expect(count).toBeLessThanOrEqual(61); }); }); + describe('getAllProvidersRateLimitStatus', () => { + it('should batch query Redis and return calculated metrics across providers', async () => { + const { getAllProvidersRateLimitStatus, clearRateLimitStatusCache } = await import( + '../../../src/processors/shared/rate-limiter.js' + ); + clearRateLimitStatusCache(); + + const now = Date.now(); + // Provider 1 (email-provider): 70 tokens, refilled now, 0 exhausted + // Provider 2 (sms-provider): 0 tokens, refilled now, 3 exhausted, last exhausted now + mockRedisClient.mget.mockResolvedValue([ + '70', + now.toString(), + '0', + null, + '0', + now.toString(), + '3', + now.toString(), + ]); + + const response = await getAllProvidersRateLimitStatus(); + + expect(response.providers.length).toBe(2); + expect(response.summary.total_providers).toBe(2); + expect(response.summary.exhausted_providers).toBe(1); + expect(response.summary.healthy_providers).toBe(1); + + const p1 = response.providers.find((p) => p.provider_id === 'email-provider'); + expect(p1).toBeDefined(); + expect(p1?.status.remaining_tokens).toBe(70); + expect(p1?.status.used_tokens).toBe(30); + expect(p1?.status.usage_percentage).toBe(30); + expect(p1?.status.is_exhausted).toBe(false); + + const p2 = response.providers.find((p) => p.provider_id === 'sms-provider'); + expect(p2).toBeDefined(); + expect(p2?.status.remaining_tokens).toBe(0); + expect(p2?.status.used_tokens).toBe(100); + expect(p2?.status.usage_percentage).toBe(100); + expect(p2?.status.is_exhausted).toBe(true); + expect(p2?.status.resets_in_ms).toBeGreaterThan(0); + expect(p2?.status.exhausted_count).toBe(3); + }); + + it('should debounce subsequent requests within CACHE_TTL_MS', async () => { + const { getAllProvidersRateLimitStatus, clearRateLimitStatusCache } = await import( + '../../../src/processors/shared/rate-limiter.js' + ); + clearRateLimitStatusCache(); + + mockRedisClient.mget.mockResolvedValue([null, null, null, null, null, null, null, null]); + + await getAllProvidersRateLimitStatus(); + await getAllProvidersRateLimitStatus(); + + // MGET should only have been called once due to in-memory cache + expect(mockRedisClient.mget).toHaveBeenCalledTimes(1); + }); + }); + describe('resetRateLimiter', () => { - it('should delete rate limiter keys from Redis', async () => { + it('should delete both hash-tagged and legacy rate limiter keys from Redis', async () => { const { resetRateLimiter } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.del.mockResolvedValue(3); + mockRedisClient.del.mockResolvedValue(5); await resetRateLimiter('email'); expect(mockRedisClient.del).toHaveBeenCalledWith( + 'ratelimit:{email}:tokens', + 'ratelimit:{email}:last_refill', + 'ratelimit:{email}:queue_position', + 'ratelimit:{email}:exhausted_count', + 'ratelimit:{email}:last_exhausted', 'ratelimit:tokens:email', 'ratelimit:last_refill:email', 'ratelimit:queue_position:email' @@ -150,30 +221,27 @@ describe('Rate Limiter Module', () => { }); describe('refillInterval normalization', () => { - // Mock the plugin registry to return different interval configs beforeEach(() => { vi.resetModules(); }); it('should normalize refillInterval: minute to per-second rate', async () => { - // Mock plugin to return minute config vi.doMock('../../../src/plugins/index.js', () => ({ getRateLimitConfig: vi.fn().mockReturnValue({ maxTokens: 60, refillRate: 60, - refillInterval: 'minute' - }) + refillInterval: 'minute', + }), })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.eval.mockResolvedValue([1, 59, 0, 0]); + mockRedisClient.eval.mockResolvedValue([1, 59, 0]); await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) - expect(callArgs[5]).toBe('60'); // maxTokens - expect(callArgs[6]).toBe('1'); // 60/minute = 1/second + expect(callArgs[6]).toBe('60'); // maxTokens + expect(callArgs[7]).toBe('1'); // 60/minute = 1/second }); it('should normalize refillInterval: hour to per-second rate', async () => { @@ -181,19 +249,18 @@ describe('Rate Limiter Module', () => { getRateLimitConfig: vi.fn().mockReturnValue({ maxTokens: 3600, refillRate: 3600, - refillInterval: 'hour' - }) + refillInterval: 'hour', + }), })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.eval.mockResolvedValue([1, 3599, 0, 0]); + mockRedisClient.eval.mockResolvedValue([1, 3599, 0]); await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) - expect(callArgs[5]).toBe('3600'); // maxTokens - expect(callArgs[6]).toBe('1'); // 3600/hour = 1/second + expect(callArgs[6]).toBe('3600'); // maxTokens + expect(callArgs[7]).toBe('1'); // 3600/hour = 1/second }); it('should normalize refillInterval: day to per-second rate', async () => { @@ -201,20 +268,18 @@ describe('Rate Limiter Module', () => { getRateLimitConfig: vi.fn().mockReturnValue({ maxTokens: 500, refillRate: 500, - refillInterval: 'day' - }) + refillInterval: 'day', + }), })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.eval.mockResolvedValue([1, 499, 0, 0]); + mockRedisClient.eval.mockResolvedValue([1, 499, 0]); await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) - expect(callArgs[5]).toBe('500'); // maxTokens - // 500/day = 500/86400 ≈ 0.005787 per second - const normalizedRate = parseFloat(callArgs[6]); + expect(callArgs[6]).toBe('500'); // maxTokens + const normalizedRate = parseFloat(callArgs[7]); expect(normalizedRate).toBeCloseTo(500 / 86400, 5); }); @@ -222,20 +287,18 @@ describe('Rate Limiter Module', () => { vi.doMock('../../../src/plugins/index.js', () => ({ getRateLimitConfig: vi.fn().mockReturnValue({ maxTokens: 100, - refillRate: 10 - // No refillInterval specified - }) + refillRate: 10, + }), })); const { consumeToken } = await import('../../../src/processors/shared/rate-limiter.js'); - mockRedisClient.eval.mockResolvedValue([1, 99, 0, 0]); + mockRedisClient.eval.mockResolvedValue([1, 99, 0]); await consumeToken('test-provider'); const callArgs = mockRedisClient.eval.mock.calls[0]; - // eval(script, numKeys=3, key1, key2, key3, arg1=maxTokens, arg2=refillRate, arg3=now) - expect(callArgs[5]).toBe('100'); // maxTokens - expect(callArgs[6]).toBe('10'); // 10/second (no normalization needed) + expect(callArgs[6]).toBe('100'); // maxTokens + expect(callArgs[7]).toBe('10'); // 10/second }); }); }); diff --git a/tests/unit/utils/backoff.utils.test.ts b/tests/unit/utils/backoff.utils.test.ts new file mode 100644 index 0000000..907efa3 --- /dev/null +++ b/tests/unit/utils/backoff.utils.test.ts @@ -0,0 +1,29 @@ +import { describe, it, expect } from 'vitest'; +import { calculateExponentialBackoff } from '../../../src/utils/backoff.utils.js'; + +describe('calculateExponentialBackoff', () => { + it('should calculate exponential backoff with default base and max delays', () => { + expect(calculateExponentialBackoff(0)).toBe(1000); // 1000 * 2^0 + expect(calculateExponentialBackoff(1)).toBe(2000); // 1000 * 2^1 + expect(calculateExponentialBackoff(2)).toBe(4000); // 1000 * 2^2 + expect(calculateExponentialBackoff(3)).toBe(8000); // 1000 * 2^3 + expect(calculateExponentialBackoff(4)).toBe(16000); // 1000 * 2^4 + expect(calculateExponentialBackoff(5)).toBe(32000); // 1000 * 2^5 + }); + + it('should handle custom baseDelay and maxDelay', () => { + expect(calculateExponentialBackoff(0, 5000, 60000)).toBe(5000); + expect(calculateExponentialBackoff(1, 5000, 60000)).toBe(10000); + expect(calculateExponentialBackoff(2, 5000, 60000)).toBe(20000); + expect(calculateExponentialBackoff(3, 5000, 60000)).toBe(40000); + expect(calculateExponentialBackoff(4, 5000, 60000)).toBe(60000); // capped at 60s + }); + + it('should cap at maxDelayMs for high retry counts', () => { + expect(calculateExponentialBackoff(20, 1000, 300000)).toBe(300000); + }); + + it('should handle negative retry counts gracefully', () => { + expect(calculateExponentialBackoff(-1)).toBe(1000); + }); +}); diff --git a/tests/unit/utils/k8s-health-probe.test.ts b/tests/unit/utils/k8s-health-probe.test.ts new file mode 100644 index 0000000..818db60 --- /dev/null +++ b/tests/unit/utils/k8s-health-probe.test.ts @@ -0,0 +1,155 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import axios from 'axios'; +import { runHealthChecks, createHealthProbeServer } from '@src/utils/k8s-health-probe.js'; +import type { HealthProbeCheck, HealthProbeServer } from '@src/types/types.js'; + +describe('k8s-health-probe utility', () => { + describe('runHealthChecks', () => { + it('should return ok: true when all checks pass', async () => { + const checks: HealthProbeCheck[] = [ + { name: 'db', check: () => true }, + { name: 'cache', check: async () => true }, + ]; + + const result = await runHealthChecks(checks); + expect(result.ok).toBe(true); + expect(result.results).toEqual({ + db: true, + cache: true, + }); + }); + + it('should return ok: false when one check fails', async () => { + const checks: HealthProbeCheck[] = [ + { name: 'db', check: () => true }, + { name: 'cache', check: () => false }, + ]; + + const result = await runHealthChecks(checks); + expect(result.ok).toBe(false); + expect(result.results).toEqual({ + db: true, + cache: false, + }); + }); + + it('should handle thrown errors gracefully and mark check false', async () => { + const checks: HealthProbeCheck[] = [ + { name: 'db', check: () => true }, + { + name: 'cache', + check: () => { + throw new Error('Connection refused'); + }, + }, + ]; + + const result = await runHealthChecks(checks); + expect(result.ok).toBe(false); + expect(result.results).toEqual({ + db: true, + cache: false, + }); + }); + + it('should handle empty checks gracefully', async () => { + const result = await runHealthChecks([]); + expect(result.ok).toBe(true); + expect(result.results).toEqual({}); + }); + }); + + describe('createHealthProbeServer', () => { + let probeServer: HealthProbeServer | null = null; + const testPort = 18991; + + afterEach(async () => { + if (probeServer) { + await probeServer.stop(); + probeServer = null; + } + }); + + it('should respond to /healthz and /readyz correctly when healthy', async () => { + const port = 18991; + probeServer = createHealthProbeServer({ + port, + serviceName: 'test-service', + livenessChecks: [{ name: 'process', check: () => true }], + readinessChecks: [{ name: 'db', check: () => true }], + }); + await probeServer.start(); + + const healthzRes = await axios.get(`http://127.0.0.1:${port}/healthz`); + expect(healthzRes.status).toBe(200); + expect(healthzRes.data).toEqual( + expect.objectContaining({ + service: 'test-service', + status: 'healthy', + checks: { process: true }, + }) + ); + + const readyzRes = await axios.get(`http://127.0.0.1:${port}/readyz`); + expect(readyzRes.status).toBe(200); + expect(readyzRes.data).toEqual( + expect.objectContaining({ + service: 'test-service', + status: 'ready', + checks: { db: true }, + }) + ); + }); + + it('should return 503 on /readyz when a readiness check fails', async () => { + const port = 18992; + probeServer = createHealthProbeServer({ + port, + serviceName: 'test-service', + readinessChecks: [ + { name: 'db', check: () => true }, + { name: 'redis', check: () => false }, + ], + }); + await probeServer.start(); + + try { + await axios.get(`http://127.0.0.1:${port}/readyz`); + expect.unreachable('Should have thrown HTTP 503'); + } catch (err: unknown) { + if (axios.isAxiosError(err)) { + expect(err.response?.status).toBe(503); + expect(err.response?.data).toEqual( + expect.objectContaining({ + service: 'test-service', + status: 'not_ready', + checks: { db: true, redis: false }, + }) + ); + } else { + throw err; + } + } + }); + + it('should return 404 for unknown endpoints', async () => { + const port = 18993; + probeServer = createHealthProbeServer({ + port, + serviceName: 'test-service', + }); + await probeServer.start(); + + try { + await axios.get(`http://127.0.0.1:${port}/unknown`); + expect.unreachable('Should have thrown HTTP 404'); + } catch (err: unknown) { + if (axios.isAxiosError(err)) { + expect(err.response?.status).toBe(404); + } else { + throw err; + } + } + }); + }); +}); diff --git a/tests/unit/workers/background.cron.test.ts b/tests/unit/workers/background.cron.test.ts new file mode 100644 index 0000000..5389394 --- /dev/null +++ b/tests/unit/workers/background.cron.test.ts @@ -0,0 +1,243 @@ +/** + * Unit Tests for Background Outbox Cron Jobs + * Tests the batch 2-phase atomic claim, eager draining loop, and cleanup routines. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import mongoose from 'mongoose'; +import { OUTBOX_STATUS } from '../../../src/types/types.js'; + +// Mock dependencies +const mockOutboxModel = { + find: vi.fn(), + updateMany: vi.fn(), + deleteMany: vi.fn(), +}; + +const mockStatusOutboxModel = { + find: vi.fn(), + updateMany: vi.fn(), +}; + +const mockBackgroundProducer = { + sendOutboxEvents: vi.fn(), + sendStatusOutboxEvents: vi.fn(), +}; + +const mockLogger = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + success: vi.fn(), +}; + +vi.mock('@src/database/models/outbox.models.js', () => ({ + default: mockOutboxModel, +})); + +vi.mock('@src/database/models/status-outbox.models.js', () => ({ + default: mockStatusOutboxModel, +})); + +vi.mock('@src/workers/producers/background.producer.js', () => ({ + sendOutboxEvents: (...args: unknown[]) => mockBackgroundProducer.sendOutboxEvents(...args), + sendStatusOutboxEvents: (...args: unknown[]) => mockBackgroundProducer.sendStatusOutboxEvents(...args), +})); + +vi.mock('@src/workers/utils/logger.js', () => ({ + cronLogger: mockLogger, +})); + +vi.mock('@src/config/env.config.js', () => ({ + env: { + WORKER_ID: 'test-worker-1', + OUTBOX_POLL_INTERVAL_MS: 5000, + OUTBOX_CLEANUP_INTERVAL_MS: 60000, + OUTBOX_BATCH_SIZE: 10, + OUTBOX_RETENTION_MS: 300000, + OUTBOX_CLAIM_TIMEOUT_MS: 30000, + }, +})); + +vi.mock('@src/config/dynamic-config.service.js', () => ({ + dynamicConfig: { + on: vi.fn(), + }, +})); + +describe('Background Cron Unit Tests', () => { + let cronModule: typeof import('../../../src/workers/cron/background.cron.js'); + + const createEmptyChain = () => ({ + sort: vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ + select: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue([]), + }), + }), + }), + }); + + beforeEach(async () => { + vi.clearAllMocks(); + mockOutboxModel.find.mockImplementation(() => createEmptyChain()); + mockStatusOutboxModel.find.mockImplementation(() => createEmptyChain()); + cronModule = await import('../../../src/workers/cron/background.cron.js'); + }); + + afterEach(async () => { + await cronModule.stopCronJobs(); + }); + + it('should manage cron job lifecycle (start, status, stop)', async () => { + expect(cronModule.isCronRunning()).toBe(false); + + cronModule.startCronJobs(); + expect(cronModule.isCronRunning()).toBe(true); + + // Calling start again should be a no-op + cronModule.startCronJobs(); + expect(cronModule.isCronRunning()).toBe(true); + + await cronModule.stopCronJobs(); + expect(cronModule.isCronRunning()).toBe(false); + }); + + it('should claim pending outbox entries using 2-phase atomic batch claim and drain eagerly', async () => { + const id1 = new mongoose.Types.ObjectId(); + const id2 = new mongoose.Types.ObjectId(); + + // Round 1: find returns 2 candidates (less than batch size 10), claiming finishes + const mockCandidates = [{ _id: id1 }, { _id: id2 }]; + const mockClaimedDocs = [ + { _id: id1, status: OUTBOX_STATUS.processing, topic: 'notification_mock' }, + { _id: id2, status: OUTBOX_STATUS.processing, topic: 'notification_mock' }, + ]; + + // Chain for outbox_model.find().sort().limit().select().lean() + const selectLeanMock = { + select: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue(mockCandidates), + }), + }; + const limitMock = { + limit: vi.fn().mockReturnValue(selectLeanMock), + }; + const sortMock = { + sort: vi.fn().mockReturnValue(limitMock), + }; + + // When finding candidates vs finding claimed entries: + mockOutboxModel.find.mockImplementation((query: Record) => { + if (query.status === OUTBOX_STATUS.pending) { + return sortMock; + } + if (query.status === OUTBOX_STATUS.processing && query.claimed_at) { + // Stale check: returns empty + return { + sort: vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ + select: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue([]), + }), + }), + }), + }; + } + // Finding claimed docs by _id: + return Promise.resolve(mockClaimedDocs); + }); + + mockOutboxModel.updateMany.mockResolvedValue({ modifiedCount: 2 }); + mockBackgroundProducer.sendOutboxEvents.mockResolvedValue({ successCount: 2, failedCount: 0 }); + + // Trigger cron jobs (which immediately runs pollOutbox) + cronModule.startCronJobs(); + await new Promise(resolve => setTimeout(resolve, 50)); + + // Verify updateMany was called atomically + expect(mockOutboxModel.updateMany).toHaveBeenCalledWith( + { _id: { $in: [id1, id2] }, status: OUTBOX_STATUS.pending }, + expect.objectContaining({ + $set: expect.objectContaining({ + status: OUTBOX_STATUS.processing, + claimed_by: 'test-worker-1', + }), + }) + ); + + // Verify sendOutboxEvents was called with the claimed docs + expect(mockBackgroundProducer.sendOutboxEvents).toHaveBeenCalledWith(mockClaimedDocs); + }); + + it('should reclaim stale entries when pending quota is remaining', async () => { + const staleId = new mongoose.Types.ObjectId(); + const staleDoc = { _id: staleId, status: OUTBOX_STATUS.processing, topic: 'notification_mock' }; + + // No pending candidates + const emptyPending = { + sort: vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ + select: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue([]), + }), + }), + }), + }; + + // Stale candidates found + const staleCandidates = { + sort: vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ + select: vi.fn().mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: staleId }]), + }), + }), + }), + }; + + mockOutboxModel.find.mockImplementation((query: Record) => { + if (query.status === OUTBOX_STATUS.pending) { + return emptyPending; + } + if (query.status === OUTBOX_STATUS.processing && query.claimed_at) { + return staleCandidates; + } + return Promise.resolve([staleDoc]); + }); + + mockOutboxModel.updateMany.mockResolvedValue({ modifiedCount: 1 }); + mockBackgroundProducer.sendOutboxEvents.mockResolvedValue({ successCount: 1, failedCount: 0 }); + + cronModule.startCronJobs(); + await new Promise(resolve => setTimeout(resolve, 50)); + + expect(mockOutboxModel.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + _id: { $in: [staleId] }, + status: OUTBOX_STATUS.processing, + }), + expect.any(Object) + ); + expect(mockBackgroundProducer.sendOutboxEvents).toHaveBeenCalledWith([staleDoc]); + }); + + it('should cleanup published events older than retention period', async () => { + vi.useFakeTimers(); + try { + mockOutboxModel.deleteMany.mockResolvedValue({ deletedCount: 5 }); + + cronModule.startCronJobs(); + // Advance time to trigger cleanup interval (60,000ms) + await vi.advanceTimersByTimeAsync(60000); + + expect(mockOutboxModel.deleteMany).toHaveBeenCalledWith( + expect.objectContaining({ + status: OUTBOX_STATUS.published, + }) + ); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/tests/unit/workers/status-consumer-health.test.ts b/tests/unit/workers/status-consumer-health.test.ts new file mode 100644 index 0000000..ce71387 --- /dev/null +++ b/tests/unit/workers/status-consumer-health.test.ts @@ -0,0 +1,94 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const eventListeners: Record void)[]> = {}; + +const kafkaConsumerMock = { + connect: vi.fn().mockResolvedValue(undefined), + subscribe: vi.fn().mockResolvedValue(undefined), + run: vi.fn().mockResolvedValue(undefined), + stop: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + commitOffsets: vi.fn().mockResolvedValue(undefined), + on: vi.fn().mockImplementation((event: string, listener: (e: unknown) => void) => { + if (!eventListeners[event]) { + eventListeners[event] = []; + } + eventListeners[event].push(listener); + }), + events: { + CRASH: 'consumer.crash', + HEARTBEAT: 'consumer.heartbeat', + STOP: 'consumer.stop', + DISCONNECT: 'consumer.disconnect', + CONNECT: 'consumer.connect', + }, +}; + +vi.mock('@src/config/kafka.config.js', () => ({ + kafka: { + consumer: vi.fn(() => kafkaConsumerMock), + }, +})); + +vi.mock('@src/database/models/notification.models.js', () => ({ + default: { + findByIdAndUpdate: vi.fn(), + }, +})); + +describe('Status Consumer Health Checks', () => { + beforeEach(() => { + vi.clearAllMocks(); + for (const key of Object.keys(eventListeners)) { + delete eventListeners[key]; + } + }); + + it('should report unhealthy when consumer is not started', async () => { + const { isStatusConsumerHealthy } = await import('@src/workers/consumers/status.consumer.js'); + expect(isStatusConsumerHealthy()).toBe(false); + }); + + it('should report healthy once consumer starts', async () => { + const { startStatusConsumer, isStatusConsumerHealthy, stopStatusConsumer } = await import( + '@src/workers/consumers/status.consumer.js' + ); + + await startStatusConsumer(); + expect(isStatusConsumerHealthy()).toBe(true); + + await stopStatusConsumer(); + expect(isStatusConsumerHealthy()).toBe(false); + }); + + it('should report unhealthy if consumer crashes', async () => { + const { startStatusConsumer, isStatusConsumerHealthy, stopStatusConsumer } = await import( + '@src/workers/consumers/status.consumer.js' + ); + + await startStatusConsumer(); + expect(isStatusConsumerHealthy()).toBe(true); + + // Emit crash event + const crashHandlers = eventListeners['consumer.crash'] || []; + for (const handler of crashHandlers) { + handler({ payload: { error: new Error('Broker disconnected') } }); + } + + expect(isStatusConsumerHealthy()).toBe(false); + + await stopStatusConsumer(); + }); + + it('should report unhealthy if heartbeat is stale', async () => { + const { startStatusConsumer, isStatusConsumerHealthy, stopStatusConsumer } = await import( + '@src/workers/consumers/status.consumer.js' + ); + + await startStatusConsumer(); + // With threshold of 0ms, heartbeat is immediately considered stale if checked with negative/0 allowance + expect(isStatusConsumerHealthy(-1)).toBe(false); + + await stopStatusConsumer(); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 83afc3e..5331216 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,11 +1,11 @@ { "compilerOptions": { "target": "ES2020", + "lib": ["ES2022"], "module": "NodeNext", "moduleResolution": "NodeNext", - "baseUrl": ".", "paths": { - "@src/*": ["src/*"] + "@src/*": ["./src/*"] }, "outDir": "./dist", "rootDir": "./src",