diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 000000000..d1316afa3 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,65 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +# TODO: delete this file after PR #1601 (pluggable database) is merged. +reviews: + # Files removed with SQLite, Typesense, the Elasticsearch query translator and the Aquarius + # routes. They no longer exist, so they need no review. + path_filters: + - '!src/@types/DDO/IDdoStateQuery.ts' + - '!src/@types/DDO/IMetadataQuery.ts' + - '!src/@types/DDO/SearchQuery.ts' + - '!src/components/database/ElasticSearchDdoStateQuery.ts' + - '!src/components/database/ElasticSearchMetadataQuery.ts' + - '!src/components/httpRoutes/aquarius.ts' + - '!schemas/**' + - '!scripts/inspect-job-metrics.mjs' + - '!typesense-compose.yml' + - '!src/@types/Typesense.ts' + - '!src/components/database/SQLLite*.ts' + - '!src/components/database/sqlite*.ts' + - '!src/components/database/Typesense*.ts' + - '!src/components/database/typesense*.ts' + - '!src/test/data/ddoSchema.ts' + - '!src/test/data/nonceSchema.ts' + - '!src/test/integration/nonce.test.ts' + - '!src/test/integration/typesense.test.ts' + - '!src/test/unit/sqliteClient.test.ts' + # More than 100 files: CodeRabbit skips the review. Docs, the lock file, agent files and + # tests that change only to fit the new database (new test files are still reviewed). + - '!docs/**' + - '!package-lock.json' + - '!AGENTS.md' + - '!CLAUDE.md' + - '!README.md' + - '!.github/copilot-instructions.md' + - '!src/test/.env.test' + - '!src/test/config.json' + - '!src/test/integration/compute.test.ts' + - '!src/test/integration/configDatabase.test.ts' + - '!src/test/integration/database.test.ts' + - '!src/test/integration/elasticsearch.test.ts' + - '!src/test/integration/imageCleanup.test.ts' + - '!src/test/integration/indexer.test.ts' + - '!src/test/integration/logs.test.ts' + - '!src/test/integration/maintenanceMode.test.ts' + - '!src/test/integration/persistentStorage.test.ts' + - '!src/test/unit/auth/token.test.ts' + - '!src/test/unit/compute.test.ts' + - '!src/test/unit/config.test.ts' + - '!src/test/unit/database/getJobsFilters.test.ts' + - '!src/test/unit/escrowFilters.test.ts' + - '!src/test/unit/fileInfoHandler.test.ts' + - '!src/test/unit/indexer/validation.test.ts' + - '!src/test/unit/maintenanceCoverage.test.ts' + - '!src/test/unit/maintenanceMode.test.ts' + - '!src/test/unit/nodeMetricsHandlers.test.ts' + - '!src/test/unit/oceanP2P.test.ts' + - '!src/test/unit/p2p/provideLimiter.test.ts' + - '!src/test/unit/p2p/republishStoredDdos.test.ts' + - '!src/test/unit/persistentStorageSharing.test.ts' + - '!src/test/unit/safeUrl.test.ts' + - '!src/test/unit/service/serviceDiskUsage.test.ts' + - '!src/test/unit/service/serviceJobsDatabase.test.ts' + - '!src/test/unit/service/serviceRestartRace.test.ts' + - '!src/test/utils/hooks.ts' + - '!src/test/utils/utils.ts' + - '!src/test/performance/util.js' diff --git a/.dockerignore b/.dockerignore index 2cc11f57e..4f295508d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,4 +15,3 @@ src/test coverage docker-compose.yml elasticsearch-compose.yml -typesense-compose.yml diff --git a/.env.example b/.env.example index dc975f4a7..77c3b6778 100644 --- a/.env.example +++ b/.env.example @@ -3,15 +3,18 @@ #check env.md file for examples and descriptions on each variable #----------------- REQUIRED -------------------------- -#This is the only required/mandatory variable -#Node will simply not run without this variable +#The node does not run without these two variables +#DB_TYPE: sqlite, postgresql or elasticsearch (the last two also need DB_URL, DB_USERNAME and DB_PASSWORD) #All the other variables can remain blank (because they have defaults) or simply commented export PRIVATE_KEY=REPLACE_ME +export DB_TYPE=sqlite #----------------------------------------------------- ## core export RPCS= export DB_URL= +export DB_USERNAME= +export DB_PASSWORD= export IPFS_GATEWAY=https://ipfs.io/ export ARWEAVE_GATEWAY=https://arweave.net/ export LOAD_INITIAL_DDOS= diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b294aa687..96e7e408b 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -39,11 +39,7 @@ export class FileInfoHandler extends CommandHandler { ### Database Abstraction Layer -Multiple database backends via `DatabaseFactory` pattern: - -- **Elasticsearch**: DDO metadata, indexer state, orders (async) -- **Typesense**: Search-optimized metadata queries -- **SQLite**: Nonce tracking, config storage, c2d Database (local) +Stores are built by `DatabaseFactory` from `DB_TYPE`: required: `sqlite` (no other setting), `postgresql` or `elasticsearch` (both need `DB_URL`, `DB_USERNAME`, `DB_PASSWORD`). SQLite and PostgreSQL share the stores in `src/components/database/sql/`; Elasticsearch uses the `Elastic*` files. The stores hold DDO metadata, indexer state, orders and the node state (nonces, config, auth tokens, buckets, node metrics, C2D jobs, service jobs, image records, service locks). Access via singleton `Database` class: `db.ddo`, `db.indexer`, `db.order`, `db.c2d`, `db.nonce`, etc. @@ -64,7 +60,7 @@ Central coordinator managing all components. Initialize with configuration and o ### Building & Testing - **Build**: `npm run build` → compiles TypeScript to `./dist/` -- **Tests**: `npm run test` runs unit + integration (requires Docker Compose for Typesense) +- **Tests**: `npm run test` runs unit + integration (integration tests need Barge; PostgreSQL or Elasticsearch only when `DB_TYPE` selects one) - **Dev**: Use `npm run build:tsc` in watch mode, or Node's `--experimental-specifier-resolution=node` for ESM imports ### Configuration @@ -72,7 +68,7 @@ Central coordinator managing all components. Initialize with configuration and o Environment-driven via `.env` or `CONFIG_PATH=/path/to/config.json`: - **PRIVATE_KEY** (required): Node identity for P2P + crypto operations -- **DB_URL**: Elasticsearch/Typesense connection +- **DB_TYPE** / **DB_URL**: database selection (required, no default) and server URL - **RPCS**: JSON mapping network IDs to RPC endpoints - **OPERATOR_SERVICE_URL**: C2D cluster URLs (array of strings) - See [docs/env.md](docs/env.md) for 40+ environment variables @@ -108,7 +104,7 @@ validateCommandParameters(command, ['required', 'fields']) // throws ValidatePar ### DDO/Asset Schema Validation -DDOs validated against SHACL schemas in [schemas/](schemas/) (v1, v3, v5, v7). Indexer enforces validation; check `ValidateDDOHandler` for signature verification. +DDOs validated against SHACL schemas from `@oceanprotocol/ddo-js` (`validateDDO` in `src/utils/asset.ts`). Indexer enforces validation; check `ValidateDDOHandler` for signature verification. ### Error Response Format diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 937767f37..371dc024e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,6 +77,49 @@ jobs: test_unit: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - db: sqlite + - db: postgresql + db_url: 'postgresql://localhost:5432/ocean' + db_username: 'ocean' + db_password: 'changeme' + - db: elasticsearch + db_url: 'http://localhost:9200' + db_username: 'elastic' + db_password: 'changeme' + # a service with an empty image does not start: each leg starts its database only + services: + postgresql: + image: ${{ matrix.db == 'postgresql' && 'postgres:17' || '' }} + env: + POSTGRES_USER: ocean + POSTGRES_PASSWORD: changeme + POSTGRES_DB: ocean + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U ocean" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + elasticsearch: + image: ${{ matrix.db == 'elasticsearch' && 'elasticsearch:8.5.1' || '' }} + env: + discovery.type: single-node + ELASTIC_PASSWORD: changeme + xpack.security.enabled: 'true' + xpack.security.http.ssl.enabled: 'false' + ES_JAVA_OPTS: -Xms512m -Xmx512m + ports: + - 9200:9200 + options: >- + --health-cmd "curl -sf -u elastic:changeme 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s'" + --health-interval 10s + --health-timeout 5s + --health-retries 20 steps: - uses: actions/checkout@v7 with: @@ -102,19 +145,51 @@ jobs: IPFS_GATEWAY: http://172.15.0.16:8080/ ARWEAVE_GATEWAY: https://arweave.net/ RPCS: '{ "1": {"rpc": "https://rpc.eth.gateway.fm", "chainId": 1, "network": "mainet", "chunkSize": 100}, "137": {"rpc": "https://polygon.meowrpc.com", "chainId": 137, "network": "polygon", "chunkSize": 100 }, "80001": {"rpc": "https://rpc-mumbai.maticvigil.com","chainId": 80001, "network": "polygon-mumbai", "chunkSize": 100 } }' - DB_URL: 'http://localhost:8108/?apiKey=xyz' - DB_TYPE: 'typesense' - DB_USERNAME: 'elastic' - DB_PASSWORD: 'changeme' + DB_TYPE: ${{ matrix.db }} + DB_URL: ${{ matrix.db_url }} + DB_USERNAME: ${{ matrix.db_username }} + DB_PASSWORD: ${{ matrix.db_password }} + # sniffing returns the container-internal address, unreachable from the runner + ELASTICSEARCH_SNIFF_ON_START: 'false' + ELASTICSEARCH_SNIFF_INTERVAL: 'false' + ELASTICSEARCH_SNIFF_ON_CONNECTION_FAULT: 'false' FEE_TOKENS: '{ "1": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", "137": "0x282d8efCe846A88B159800bd4130ad77443Fa1A1", "80001": "0xd8992Ed72C445c35Cb4A2be468568Ed1079357c8", "56": "0xDCe07662CA8EbC241316a15B611c89711414Dd1a" }' FEE_AMOUNT: '{ "amount": 1, "unit": "MB" }' - uses: actions/upload-artifact@v7 with: - name: coverage + name: coverage-unit-${{ matrix.db }} path: coverage/ test_integration: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - db: sqlite + - db: postgresql + db_url: 'postgresql://localhost:5432/ocean' + db_username: 'ocean' + db_password: 'changeme' + # the Elasticsearch of Barge + - db: elasticsearch + db_url: 'http://localhost:9200' + db_username: 'elastic' + db_password: 'changeme' + services: + postgresql: + image: ${{ matrix.db == 'postgresql' && 'postgres:17' || '' }} + env: + POSTGRES_USER: ocean + POSTGRES_PASSWORD: changeme + POSTGRES_DB: ocean + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U ocean" + --health-interval 5s + --health-timeout 5s + --health-retries 20 steps: - uses: actions/checkout@v7 with: @@ -148,7 +223,7 @@ jobs: - name: Run Barge working-directory: ${{ github.workspace }}/barge run: | - bash -x start_ocean.sh --no-node --with-typesense 2>&1 > start_ocean.log & + bash -x start_ocean.sh --no-node 2>&1 > start_ocean.log & - run: npm ci - run: npm run build - run: docker image ls @@ -163,7 +238,7 @@ jobs: [ -f "$HOME/.ocean/ocean-contracts/artifacts/ready" ] && break done - name: docker logs - run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1 + run: docker logs ocean-ocean-contracts-1 if: ${{ failure() }} - name: Build Docker registry auths id: docker_auths @@ -191,19 +266,21 @@ jobs: IPFS_GATEWAY: http://172.15.0.16:8080/ ARWEAVE_GATEWAY: https://arweave.net/ RPCS: '{ "8996": {"rpc": "http://127.0.0.1:8545", "chainId": 8996, "network": "development", "chunkSize": 100}}' - DB_URL: 'http://localhost:9200' - DB_TYPE: 'elasticsearch' + DB_TYPE: ${{ matrix.db }} + DB_URL: ${{ matrix.db_url }} + DB_USERNAME: ${{ matrix.db_username }} + DB_PASSWORD: ${{ matrix.db_password }} FEE_TOKENS: '{ "1": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", "137": "0x282d8efCe846A88B159800bd4130ad77443Fa1A1", "80001": "0xd8992Ed72C445c35Cb4A2be468568Ed1079357c8", "56": "0xDCe07662CA8EbC241316a15B611c89711414Dd1a" }' FEE_AMOUNT: '{ "amount": 1, "unit": "MB" }' ASSET_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-assets.json' ACCOUNT_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-accounts.json' DOCKER_REGISTRY_AUTHS: ${{ steps.docker_auths.outputs.value }} - name: docker logs - run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1 + run: docker logs ocean-ocean-contracts-1 if: ${{ failure() }} - uses: actions/upload-artifact@v7 with: - name: coverage + name: coverage-integration-${{ matrix.db }} path: coverage/ test_system: @@ -250,7 +327,7 @@ jobs: - name: Run Barge working-directory: ${{ github.workspace }}/barge run: | - bash -x start_ocean.sh --no-node --with-typesense 2>&1 > start_ocean.log & + bash -x start_ocean.sh --no-node 2>&1 > start_ocean.log & - run: docker image ls - name: Delete default runner images run: | @@ -260,13 +337,16 @@ jobs: with: persist-credentials: false repository: 'oceanprotocol/ocean-cli' - ref: 'feature/subsidy_provider' + # the CLI and ocean.js without Aquarius (/api/query, /api/ddo); back to the default + # branches when both are merged + ref: 'feat/node-neutral-query' path: 'ocean-cli' - name: Checkout Ocean-js uses: actions/checkout@v7 with: persist-credentials: false repository: 'oceanprotocol/ocean.js' + ref: 'feat/node-neutral-query' path: 'ocean.js' - name: Build ocean-js working-directory: ${{ github.workspace }}/ocean.js @@ -302,7 +382,7 @@ jobs: done - name: docker logs - run: docker logs ocean-contracts-1 && docker logs ocean-typesense-1 + run: docker logs ocean-contracts-1 if: ${{ failure() }} - name: Build Docker registry auths @@ -325,6 +405,7 @@ jobs: run: | npm run start > ocean-node.log 2>&1 & env: + DB_TYPE: sqlite PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} IPFS_GATEWAY: http://172.15.0.16:8080/ ARWEAVE_GATEWAY: https://arweave.net/ @@ -332,16 +413,12 @@ jobs: HTTP_API_PORT: 8001 RPCS: '{ "8996": {"rpc": "http://127.0.0.1:8545", "chainId": 8996, "network": "development", "chunkSize": 100} }' INDEXER_NETWORKS: '[8996]' - DB_URL: 'http://localhost:9200' FEE_TOKENS: '{ "1": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", "137": "0x282d8efCe846A88B159800bd4130ad77443Fa1A1", "80001": "0xd8992Ed72C445c35Cb4A2be468568Ed1079357c8", "56": "0xDCe07662CA8EbC241316a15B611c89711414Dd1a" }' FEE_AMOUNT: '{ "amount": 1, "unit": "MB" }' AUTHORIZED_DECRYPTERS: '["0xe2DD09d719Da89e5a3D0F2549c7E24566e947260"]' P2P_ENABLE_UPNP: 'false' P2P_ENABLE_AUTONAT: 'false' ALLOWED_ADMINS: '["0xe2DD09d719Da89e5a3D0F2549c7E24566e947260"]' - DB_TYPE: 'elasticsearch' - DB_USERNAME: 'elastic' - DB_PASSWORD: 'changeme' MAX_REQ_PER_MINUTE: 320 MAX_CONNECTIONS_PER_MINUTE: 320 DOCKER_COMPUTE_ENVIRONMENTS: '[{"socketPath":"/var/run/docker.sock","environments":[{"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"resources":[{"id":"cpu","total":4,"max":4,"min":1,"type":"cpu"},{"id":"ram","total":10,"max":10,"min":1,"type":"ram"},{"id":"disk","total":10,"max":10,"min":0,"type":"disk"}],"fees":{"8996":[{"prices":[{"id":"cpu","price":1}]}]},"free":{"maxJobDuration":60,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1},{"id":"disk","max":1}]}}]}]' @@ -377,6 +454,5 @@ jobs: fi echo "=== Docker Logs ===" docker logs ocean-contracts-1 || echo "No ocean-contracts-1 container" - docker logs ocean-typesense-1 || echo "No ocean-typesense-1 container" echo "=== System Processes ===" ps aux | grep -E "(node|ocean)" || echo "No node/ocean processes found" diff --git a/.gitignore b/.gitignore index 8da551d01..155fa0e2a 100644 --- a/.gitignore +++ b/.gitignore @@ -133,9 +133,6 @@ out .yarn/install-state.gz .pnp.* -# Typesense -typesense-data - # IDEA .idea diff --git a/AGENTS.md b/AGENTS.md index 11d37232a..11dfced4d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,16 +23,16 @@ libp2p P2P interface, both of which dispatch to the same set of command handlers test command. If the pinned version is already installed directly (no nvm available), verifying the active runtime against `.nvmrc` instead is sufficient — e.g. `test "v$(cat .nvmrc)" = "$(node -v)"`; do not abort the build just because `nvm` is - absent. The wrong Node version fails with errors like `Unexpected token 'with'`, - missing `GLIBC_2.38`, or — since the SQLite layer uses the built-in `node:sqlite` - module — `ERR_UNKNOWN_BUILTIN_MODULE: node:sqlite` on Node < 22.13. This is enforced by + absent. The wrong Node version fails with errors like `Unexpected token 'with'` or + missing `GLIBC_2.38`. This is enforced by `.cursor/rules/tests-nvm.mdc` and this `AGENTS.md`. - **There is no `postinstall` step.** `npm install` is plain. (Historically a `postinstall` ran `scripts/fix-libp2p-http-utils.js` to default a missing URL port to 443/80 in `@libp2p/http-utils`; upstream shipped that fix in `2.0.3`, so both the hook and the script were removed.) -- **Docker + docker-compose** are needed for the metadata database (Typesense or - Elasticsearch) and for C2D (Compute-to-Data) via the local Docker socket. +- **Docker + docker-compose** are needed for C2D (Compute-to-Data) via the local Docker + socket, and for PostgreSQL or Elasticsearch when `DB_TYPE` selects one of them. SQLite + needs no container. - TypeScript config: **TypeScript 6**, ESM (`module: esnext`, `target: ES2022`, `moduleResolution: node`), `experimentalDecorators` + `emitDecoratorMetadata` enabled, `rootDir: ./src`, `outDir: ./dist`. Most local imports use the `.js` extension @@ -45,12 +45,13 @@ libp2p P2P interface, both of which dispatch to the same set of command handlers ~123 errors), and `typescript-eslint` has to support TS 7 — its peer range is `>=4.8.4 <6.1.0`, so it pins us below 7 today. -### Mandatory config: `PRIVATE_KEY` + a config file +### Mandatory config: `PRIVATE_KEY`, `DB_TYPE` and a config file -`PRIVATE_KEY` (with the `0x` prefix, 66 chars) is the **only** required environment -variable — it seeds the node identity (libp2p peerId), the EVM signer, and encryption keys. +`PRIVATE_KEY` (with the `0x` prefix, 66 chars) and `DB_TYPE` are the required environment +variables — `PRIVATE_KEY` seeds the node identity (libp2p peerId), the EVM signer, and +encryption keys; `DB_TYPE` selects the database. A JSON config file must also exist (see §3); every other setting has a default or comes -from that file / env overrides. Generate a key with: +from that file / env overrides. `DB_TYPE=sqlite` needs no other setting. Generate a key with: ```bash node dist/helpers/scripts/generatePK.js # prints a key/address to stdout @@ -139,8 +140,9 @@ the compiled output. In tests, do **not** mutate `process.env` directly — use **Integration tests require a running local chain + services (Barge).** Clone `oceanprotocol/barge`, `git checkout feature/nodes`, then `./start_ocean.sh -with-c2d` -(see `docs/testing.md` and `docs/database.md` for the exact flags per DB type). The default -`config.json` points the DB at Typesense on `http://localhost:8108`. +(see `docs/testing.md` and `docs/database.md` for the exact flags). The tests run on the +database that `DB_TYPE` selects (the test hooks set `sqlite` when it is not set); suites that +use the Elasticsearch client directly are wrapped in `describeOnElasticsearch`. Other useful scripts: `npm run client` (2-node download flow demo), `npm run check-nonce` (nonce tracking; needs DB), and k6 perf tests `test:smoke` / `test:load` / `test:stress` / @@ -159,8 +161,9 @@ the files in `logs/` instead (see `docs/Logs.md`). ### Databases for local dev ```bash -docker-compose -f typesense-compose.yml up -d # Typesense (default in config.json) -docker-compose -f elasticsearch-compose.yml up -d # Elasticsearch alternative +# DB_TYPE=sqlite needs nothing. For the other two: +docker-compose -f postgresql-compose.yml up -d # DB_TYPE=postgresql +docker-compose -f elasticsearch-compose.yml up -d # DB_TYPE=elasticsearch ``` --- @@ -182,28 +185,15 @@ via `src/utils/config/index.ts` and `src/utils/index.ts`). Key facts: still resolve a home directory. If it is missing, startup fails with `Config file not found at path: ...`, even when every setting is given via env. The repo - ships a ready `config.json` (Typesense DB, chain `8996`/development, HTTP port `8001`, - single Docker C2D env). + ships a ready `config.json` (chain `8996`/development, HTTP port `8001`, single Docker C2D + env). - **`INTERFACES`** env var (JSON array like `["HTTP","P2P"]`) toggles `hasHttp` / `hasP2P`. - Omitted = both enabled. `DB_URL`/`DB_TYPE` presence drives `hasIndexer` — without a valid - metadata DB config the Indexer and the metadata-backed features (DDOs, orders, access - lists, escrow events, DB logs) are disabled, but the SQLite core still runs: nonce, auth - tokens, C2D jobs, node config (and best-effort node metrics). So nonce and auth services - keep working. This graceful-degrade path only applies when the config itself is - schema-valid: `dbConfig.dbType` is `z.string().nullable()` (`schemas.ts`), so a non-string - value (number/bool/object) fails zod validation in `buildMergedConfig()` and aborts the - **whole node** at startup — it does not disable just the metadata DB. Because the field is - `.nullable()` and **not** `.optional()`, `dbType` must be **present**: `null` is valid and - selects the no-metadata-DB mode above, but **omitting the key** from a supplied `dbConfig` - (e.g. a config file with `DB_URL` unset) is undefined and also fails zod and aborts the - whole node. Use `dbType: null`, not an absent key, for the no-DB mode. - - **Paid/free compute still needs the metadata DB to resolve a published dataset.** SQLite - preserves C2D **job storage**, but `FindDdoHandler.findAndFormatDdo` (`ddoHandler.ts`) - falls back to the P2P `FIND_DDO` lookup when no local DDO DB exists, and that network - response is a **summary** (`{id,lastUpdateTx,lastUpdateTime,provider}`), not the full - DDO. Parsing it for `services`/`indexedMetadata` throws and returns `null`, so - `initialize`/`startCompute` cannot resolve the asset. "C2D keeps working" means the job - runtime, not compute against a published DID without a metadata DB. + Omitted = both enabled. `DB_TYPE` is required and selects the database: `sqlite`, `postgresql` + or `elasticsearch`; with no `DB_TYPE` config validation fails and the node does not start + (no default, no guess from `DB_URL`). The last two require `DB_URL`, `DB_USERNAME` and + `DB_PASSWORD` (or `dbConfig` in the file); the URL is not checked, the driver connects to + it as given. An empty `DB_*` value counts as not set. Node state does not move between + databases. There is no DEGRADED / no-DB mode. - The full catalog of env vars is defined as `ENVIRONMENT_VARIABLES` in `src/utils/constants.ts` and documented in `docs/env.md` (long file — the authoritative reference). `.env.example` lists them grouped as core / p2p / compute. @@ -211,7 +201,7 @@ via `src/utils/config/index.ts` and `src/utils/index.ts`). Key facts: logging (config is read from many places; there's a known TODO to centralize access on the `OceanNode` class). -Selected env vars worth knowing: `RPCS` (per-chain RPC map, JSON), `DB_URL`/`DB_TYPE`, +Selected env vars worth knowing: `RPCS` (per-chain RPC map, JSON), `DB_URL`, `ALLOWED_ADMINS`(+`_LIST`) (gate admin commands), `AUTHORIZED_DECRYPTERS`/`_PUBLISHERS`/ `ALLOWED_VALIDATORS` (+ their access-list variants), `MAX_REQ_PER_MINUTE` (rate limit), `RATE_DENY_LIST`, `HTTP_API_PORT`, `JWT_SECRET`, `OPERATOR_SERVICE_URL` (external C2D @@ -329,13 +319,14 @@ Handler source is grouped under `src/components/core/`: the `Escrow` component; `serviceResourceMatching.ts` maps requested cpu/ram/disk/gpu against environment pools (dual-gate: per-env ceiling + engine-wide pool; GPUs tracked globally). See `docs/compute.md`. -- **database/** — `Database.init()` factory (`index.ts`, `DatabaseFactory.ts`). The metadata - DB backend is pluggable: **Typesense or Elasticsearch** (chosen by `DB_TYPE`) for DDOs, - indexer state, logs, orders, ddoState, access lists, escrow events — behind the - `Abstract*Database` interfaces in `BaseDatabase.ts`. **SQLite** is always used for the - nonce DB, config DB, C2D job DB, and auth-token DB (works even with no metadata DB - configured) — via Node's built-in `node:sqlite` module (no native addon), wrapped by - `SqliteClient` in `src/components/database/sqliteClient.ts`. See `docs/database.md`. +- **database/** — `Database.init()` gets the stores from `DatabaseFactory.createStores`, the + only place that selects the backend from `dbConfig.dbType`. The stores hold the index data + (DDOs, indexer state, logs, orders, ddoState, access lists, escrow events — behind the + `Abstract*Database` classes in `BaseDatabase.ts`) and the node state (nonces, config, auth + tokens, buckets, node metrics, C2D jobs, service jobs, image records, service locks). + Backends: **SQLite** and **PostgreSQL** share the stores in `database/sql/` + (`SqlClient`); **Elasticsearch** uses the `Elastic*` files (mappings in + `ElasticSchemas.ts`). Node metrics are kept per `INSTANCE_ID`. See `docs/database.md`. - **KeyManager/** — provider-abstraction over the node key (`docs/KeyManager.md`). Currently `RawPrivateKeyProvider` (from `PRIVATE_KEY`); derives the libp2p peerId/keys and the EVM address, and caches the ethers signer. Designed to add KMS providers (GCP/AWS) later. @@ -360,9 +351,8 @@ Handler source is grouped under `src/components/core/`: `constants.ts`, `crypt.ts` (hashing/signing), `blockchain.ts`, `logging/` (winston: console, per-module rotating files in `logs/` by default when `LOG_DB` is off, optional DB transport), `cronjobs/`, `validators.ts`, `credentials.ts`, `accessList.ts`, `asset.ts`, `attestation.ts`. -- Runtime data dirs: `databases/` (SQLite files + libp2p LevelDB store), `c2d_storage/` (C2D - job working data), `logs/`, `schemas/` (SHACL DDO validation schemas — shipped into the - Docker image). Service-on-demand templates are not shipped: the node reads +- Runtime data dirs: `databases/` (only `p2p-store`, the libp2p LevelDB store), + `c2d_storage/` (C2D job working data), `logs/`. Service-on-demand templates are not shipped: the node reads them from the folder given by `serviceTemplatesPath` / `SERVICE_TEMPLATES_PATH`, which the operator mounts in at run time. - `tsoa.json` configures OpenAPI spec generation from `src/components/httpRoutes/**`; the @@ -374,7 +364,7 @@ Handler source is grouped under `src/components/core/`: Multi-stage `Dockerfile` (builder + runner), both stages on `node:26.12.1-trixie-slim` pinned by digest. The runner ships only -`dist/`, `node_modules`, `schemas/`, `hfpatch/` (the `sitecustomize.py` the node copies +`dist/`, `node_modules`, `hfpatch/` (the `sitecustomize.py` the node copies into services — see `docs/services.md`) and `config.json` (`.dockerignore` excludes all of `docs/`) — no service templates. It exposes P2P ports `9000-9003,9005` and HTTP `8000`. `docker-entrypoint.sh` handles Docker socket group membership at runtime so C2D diff --git a/CLAUDE.md b/CLAUDE.md index 1ccc31187..cf73f5d97 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,22 +12,22 @@ of which dispatch to the same set of command handlers. ## 1. Environment & tooling prerequisites -- **Node.js 26 is required** (`.nvmrc` pins 26.12.1`— the current Node 26 LTS — -matching the Dockerfile and CI;`package.json` `engines`requires`>=24`, so Node 22 is -no longer a supported runtime). The Dockerfile additionally pins each base -image by digest, so bumping the version means updating the `sha256:`alongside the tag — -resolve it with`docker buildx imagetools inspect node:-trixie`, not from the tag -alone. Always run `nvm use`(or`source ~/.nvm/nvm.sh && nvm use`) before -any `npm`, build, or test command. The wrong Node version fails with errors like -`Unexpected token 'with'`, missing `GLIBC_2.38`, or — since the SQLite layer uses the -built-in `node:sqlite`module —`ERR_UNKNOWN_BUILTIN_MODULE: node:sqlite`on Node < 22.13. -This is enforced by`.cursor/rules/tests-nvm.mdc`and the in-repo`CLAUDE.md`. +- **Node.js 26 is required** (`.nvmrc` pins `26.12.1`, the current Node 26 LTS, matching + the Dockerfile and CI; `package.json` `engines` requires `>=24`, so Node 22 is no longer a + supported runtime). The Dockerfile also pins each base image by digest, so a version bump + updates the `sha256:` with the tag: resolve it with + `docker buildx imagetools inspect node:-trixie`, not from the tag alone. Always + run `nvm use` (or `source ~/.nvm/nvm.sh && nvm use`) before any `npm`, build or test + command. The wrong Node version fails with errors like `Unexpected token 'with'` or a + missing `GLIBC_2.38`. This is enforced by `.cursor/rules/tests-nvm.mdc` and the in-repo + `CLAUDE.md`. - **There is no `postinstall` step.** `npm install` is plain. (Historically a `postinstall` ran `scripts/fix-libp2p-http-utils.js` to default a missing URL port to 443/80 in `@libp2p/http-utils`; upstream shipped that fix in `2.0.3`, so both the hook and the script were removed.) -- **Docker + docker-compose** are needed for the metadata database (Typesense or - Elasticsearch) and for C2D (Compute-to-Data) via the local Docker socket. +- **Docker + docker-compose** are needed for C2D (Compute-to-Data) via the local Docker + socket, and for PostgreSQL or Elasticsearch when `DB_TYPE` selects one of them. SQLite + needs no container. - TypeScript config: **TypeScript 6**, ESM (`module: esnext`, `target: ES2022`, `moduleResolution: node`), `experimentalDecorators` + `emitDecoratorMetadata` enabled, `rootDir: ./src`, `outDir: ./dist`. Most local imports use the `.js` extension @@ -40,11 +40,12 @@ This is enforced by`.cursor/rules/tests-nvm.mdc`and the in-repo`CLAUDE.md`. ~123 errors), and `typescript-eslint` has to support TS 7 — its peer range is `>=4.8.4 <6.1.0`, so it pins us below 7 today. -### Only-mandatory config: `PRIVATE_KEY` +### Mandatory config: `PRIVATE_KEY` and `DB_TYPE` -`PRIVATE_KEY` (with the `0x` prefix) is the **only** required environment variable — it -seeds the node identity (libp2p peerId), the EVM signer, and encryption keys. Everything -else has defaults. Generate one with: +`PRIVATE_KEY` (with the `0x` prefix) and `DB_TYPE` are the required environment variables. +`PRIVATE_KEY` seeds the node identity (libp2p peerId), the EVM signer, and encryption keys; +`DB_TYPE` selects the database (`sqlite` needs no other setting). Everything else has +defaults. Generate one with: ```bash node dist/helpers/scripts/generatePK.js # prints a key/address to stdout @@ -133,8 +134,12 @@ the compiled output. In tests, do **not** mutate `process.env` directly — use **Integration tests require a running local chain + services (Barge).** Clone `oceanprotocol/barge`, `git checkout feature/nodes`, then `./start_ocean.sh -with-c2d` -(see `docs/testing.md` and `docs/database.md` for the exact flags per DB type). The default -`config.json` points the DB at Typesense on `http://localhost:8108`. +(see `docs/testing.md` and `docs/database.md` for the exact flags). The tests run on the +database that `DB_TYPE` selects (the test hooks set `sqlite` when it is not set; note that +`src/test/.env.test` can set `DB_URL`); suites that use the +Elasticsearch client directly are wrapped in `describeOnElasticsearch` and run only with +`DB_TYPE=elasticsearch`. The SQL store suites (`src/test/unit/database/`) also run on +PostgreSQL with `DB_TYPE=postgresql`. Other useful scripts: `npm run client` (2-node download flow demo), `npm run check-nonce` (nonce tracking; needs DB), `npm run logs` (calls the signed `/logs` API; currently broken, see `docs/Logs.md`), and k6 perf tests @@ -144,8 +149,9 @@ a running node). ### Databases for local dev ```bash -docker-compose -f typesense-compose.yml up -d # Typesense (default in config.json) -docker-compose -f elasticsearch-compose.yml up -d # Elasticsearch alternative +# DB_TYPE=sqlite needs nothing. For the other two: +docker-compose -f postgresql-compose.yml up -d # DB_TYPE=postgresql +docker-compose -f elasticsearch-compose.yml up -d # DB_TYPE=elasticsearch ``` --- @@ -157,10 +163,14 @@ via `src/utils/config/index.ts` and `src/utils/index.ts`). Key facts: - **Two config sources.** Either environment variables, or a JSON file. `getConfigFilePath()` resolves the file from `CONFIG_PATH` env var, else `./config.json` in cwd. The repo ships a - ready `config.json` (Typesense DB, chain `8996`/development, single Docker C2D env). + ready `config.json` (chain `8996`/development, single Docker C2D env). - **`INTERFACES`** env var (JSON array like `["HTTP","P2P"]`) toggles `hasHttp` / `hasP2P`. - Omitted = both enabled. `DB_URL`/`DB_TYPE` presence drives `hasIndexer` — without a valid - DB config the Indexer is disabled and only the SQLite nonce DB is available. + Omitted = both enabled. `DB_TYPE` is required and selects the database: `sqlite`, `postgresql` + or `elasticsearch`; with no `DB_TYPE` config validation fails and the node does not start + (no default, no guess from `DB_URL`). The last two require `DB_URL`, `DB_USERNAME` and + `DB_PASSWORD` (or `dbConfig` in the file); the URL is not checked, the driver connects to + it as given. An empty `DB_*` value counts as not set. Node state does not move between + databases. There is no DEGRADED / no-DB mode. - The full catalog of env vars is defined as `ENVIRONMENT_VARIABLES` in `src/utils/constants.ts` and documented in `docs/env.md` (long file — the authoritative reference). `.env.example` lists them grouped as core / p2p / compute. @@ -168,7 +178,7 @@ via `src/utils/config/index.ts` and `src/utils/index.ts`). Key facts: logging (config is read from many places; there's a known TODO to centralize access on the `OceanNode` class). -Selected env vars worth knowing: `RPCS` (per-chain RPC map, JSON), `DB_URL`/`DB_TYPE`, +Selected env vars worth knowing: `RPCS` (per-chain RPC map, JSON), `DB_URL`, `ALLOWED_ADMINS`(+`_LIST`) (gate admin commands), `AUTHORIZED_DECRYPTERS`/`_PUBLISHERS`/ `ALLOWED_VALIDATORS` (+ their access-list variants), `MAX_REQ_PER_MINUTE` (rate limit), `RATE_DENY_LIST`, `HTTP_API_PORT`, `JWT_SECRET`, `OPERATOR_SERVICE_URL` (external C2D @@ -220,7 +230,7 @@ Multiple front doors, one shared `handler.handle(task)` primitive: - **P2P inbound** (`src/components/P2P/handleProtocolCommands.ts`): a libp2p protocol handler reads a length-prefixed command frame off the stream, applies connection/request rate limits, and dispatches through the same `CoreHandlersRegistry`. -- **RESTful routes** (`src/components/httpRoutes/*`, e.g. `provider.ts`, `aquarius.ts`, +- **RESTful routes** (`src/components/httpRoutes/*`, e.g. `provider.ts`, `ddo.ts`, `compute.ts`, `auth.ts`, `escrow.ts`, `accessList.ts`, `persistentStorage.ts`) are ergonomic wrappers that build the equivalent `Command` and invoke the corresponding handler. Route mounting lives in `src/components/httpRoutes/index.ts` @@ -289,13 +299,25 @@ Handler source is grouped under `src/components/core/`: the `Escrow` component; `serviceResourceMatching.ts` maps requested cpu/ram/disk/gpu against environment pools (dual-gate: per-env ceiling + engine-wide pool; GPUs tracked globally). See `docs/compute.md`. -- **database/** — `Database.init()` factory (`index.ts`, `DatabaseFactory.ts`). The metadata - DB backend is pluggable: **Typesense or Elasticsearch** (chosen by `DB_TYPE`) for DDOs, - indexer state, logs, orders, ddoState, access lists, escrow events — behind the - `Abstract*Database` interfaces in `BaseDatabase.ts`. **SQLite** is always used for the - nonce DB, config DB, C2D job DB, and auth-token DB (works even with no metadata DB - configured) — via Node's built-in `node:sqlite` module (no native addon), wrapped by - `SqliteClient` in `src/components/database/sqliteClient.ts`. See `docs/database.md`. +- **database/** — `Database.init()` gets the 13 stores (`DatabaseStores`) from + `DatabaseFactory.createStores`, the only place that selects the backend from + `dbConfig.dbType`. No file outside this folder imports a backend. The stores hold the + index data (DDOs, indexer state, logs, orders, ddoState, access lists, escrow events — + behind the `Abstract*Database` classes in `BaseDatabase.ts`) and the node state (nonces, + config, auth tokens, buckets, node metrics, C2D jobs, service jobs, image records, service + locks — behind the provider interfaces). Backends: + - **SQLite** and **PostgreSQL**: one set of stores in `database/sql/`, written + against the `SqlClient` interface (`SqliteClient` on `node:sqlite`, `PostgresClient` on + `pg`). The rules for SQL that runs on both are at the top of `sql/SqlClient.ts`. + `SqlDocuments` keeps JSON documents with a field table and runs typed conditions + (`Condition` in `BaseDatabase.ts`) as fixed SQL. `SqliteStores.ts` keeps the file + names of the older releases and upgrades their tables. + - **Elasticsearch**: the `Elastic*` files (index mappings in `ElasticSchemas.ts`). + - The DDO search (`query` command, `POST /api/query`) is typed: + `assetQuery.ts` checks the request (zod) and turns the filters into conditions; SQL + and Elasticsearch (`esConditions.ts`) each run them with fixed queries. There is no + Elasticsearch query syntax on any database. + - Node metrics are kept per `INSTANCE_ID`. See `docs/database.md`. - **KeyManager/** — provider-abstraction over the node key (`docs/KeyManager.md`). Currently `RawPrivateKeyProvider` (from `PRIVATE_KEY`); derives the libp2p peerId/keys and the EVM address, and caches the ethers signer. Designed to add KMS providers (GCP/AWS) later. @@ -316,9 +338,9 @@ Handler source is grouped under `src/components/core/`: `constants.ts`, `crypt.ts` (hashing/signing), `blockchain.ts`, `logging/` (winston: console, per-module rotating files in `logs/` by default when `LOG_DB` is off, optional DB transport), `cronjobs/`, `validators.ts`, `credentials.ts`, `accessList.ts`, `asset.ts`, `attestation.ts`. -- Runtime data dirs: `databases/` (SQLite files + libp2p LevelDB store), `c2d_storage/` (C2D - job working data), `logs/`, `schemas/` (SHACL DDO validation schemas — shipped into the - Docker image). Service-on-demand templates are not shipped: the node reads +- Runtime data dirs: `databases/` (`p2p-store`, the libp2p LevelDB store, and the SQLite + files of the node with the default database), + `c2d_storage/` (C2D job working data), `logs/`. Service-on-demand templates are not shipped: the node reads them from the folder given by `serviceTemplatesPath` / `SERVICE_TEMPLATES_PATH`, which the operator mounts in at run time. - `tsoa.json` configures OpenAPI spec generation from `src/components/httpRoutes/**`; the @@ -329,7 +351,7 @@ Handler source is grouped under `src/components/core/`: ## 5. Docker & deployment Multi-stage `Dockerfile` (builder + slim runner) on `node:24`. The runner ships only -`dist/`, `node_modules`, `schemas/`, `hfpatch/` (the `sitecustomize.py` the node copies +`dist/`, `node_modules`, `hfpatch/` (the `sitecustomize.py` the node copies into services — see `docs/services.md`) and `config.json` (`.dockerignore` excludes all of `docs/`) — no service templates. It exposes P2P ports `9000-9003,9005` and HTTP `8000`. `docker-entrypoint.sh` handles Docker socket group membership at runtime so C2D diff --git a/Dockerfile b/Dockerfile index 30367ae8f..c72c9117a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,17 +40,17 @@ WORKDIR /usr/src/app COPY --chown=node:node --from=builder /usr/src/app/dist ./dist COPY --chown=node:node --from=builder /usr/src/app/node_modules ./node_modules -COPY --chown=node:node --from=builder /usr/src/app/schemas ./schemas # sitecustomize.py the node copies into service containers (src/components/c2d/serviceHfMirror.ts) COPY --chown=node:node --from=builder /usr/src/app/hfpatch ./hfpatch COPY --chown=node:node --from=builder /usr/src/app/package.json ./ COPY --chown=node:node --from=builder /usr/src/app/config.json ./ -# `databases` holds everything the node must not lose across a restart: the SQLite files -# (nonce, config, C2D jobs, auth tokens) and `databases/p2p-store`, the LevelDatastore that -# backs libp2p. kad-dht's reprovider refreshes this node's provider records from that -# datastore, so if it starts empty the node stops answering for content it still holds, and the -# records expire out of the network without anything noticing. The directory is created and +# `databases` holds what the node must not lose across a restart on this host: +# `databases/p2p-store`, the LevelDatastore that backs libp2p, and the SQLite files of the +# node when it uses the default database. kad-dht's reprovider refreshes this node's +# provider records from that datastore, so if it starts empty the node stops answering for +# content it still holds, and the records expire out of the network without anything +# noticing. The directory is created and # handed to the unprivileged `node` user here, and VOLUME declares the mount point so a named # volume or bind mount can be attached; docker-entrypoint.sh re-applies ownership at runtime # for the bind-mount case, where the host directory arrives owned by whoever created it. diff --git a/README.md b/README.md index 750a0c3b3..f5f60ff4a 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ directory (indexed at the bottom) for the detailed references. Ocean Node has **two front doors and one dispatcher**: -- **HTTP** — `POST /directCommand` (plus ergonomic REST routes under `provider`, `aquarius`, +- **HTTP** — `POST /directCommand` (plus ergonomic REST routes under `provider`, `ddo`, `query`, `compute`, `auth`, `escrow`, …) normalizes each request into a `Command`. - **P2P** — a libp2p protocol handler reads length-prefixed command frames off the stream. @@ -51,10 +51,10 @@ See [Architecture](docs/Arhitecture.md) for the layered model and Mermaid diagra - **OS**: latest LTS Ubuntu or latest macOS recommended; other OSes (including Windows) should also work. -A **persistent `databases/` volume is REQUIRED**, not optional. It holds the SQLite files -(nonce, config, C2D jobs, auth tokens) and `databases/p2p-store`, the libp2p datastore that -backs this node's DHT provider records. Starting it empty silently drops content the node -still serves. The Dockerfile declares `VOLUME ["/usr/src/app/databases"]` for this reason — +A **persistent `databases/` volume is REQUIRED**, not optional. It holds +`databases/p2p-store`, the libp2p datastore that backs this node's DHT provider records, and, +with `DB_TYPE=sqlite`, all the data of the node. Starting it empty silently drops +content the node still serves. The Dockerfile declares `VOLUME ["/usr/src/app/databases"]` for this reason — see "Persistent node data" in [Docker deployment](docs/dockerDeployment.md). ## Quick start (operators) @@ -90,24 +90,21 @@ PM2 is a process manager that makes it easy to run and monitor Node.js apps. ```bash npm install -g pm2 -# 1. Start a metadata database (Typesense is the default in config.json) -docker-compose -f typesense-compose.yml up -d - -# 2. Install deps and build — `npm run start` runs dist/index.js, so a prior build is required +# 1. Install deps and build — `npm run start` runs dist/index.js, so a prior build is required npm install npm run build -# 3. Set up env vars. The node does NOT auto-load .env at runtime (dotenv is only used in +# 2. Set up env vars. The node does NOT auto-load .env at runtime (dotenv is only used in # tests), so the vars must be in the shell environment before `pm2 start`. # Interactive helper, then load the generated .env into this shell: npm run setupEnv source .env -# …or set them manually (PRIVATE_KEY is the only mandatory one; keep the 0x prefix): +# …or set them manually (only PRIVATE_KEY is mandatory; keep the 0x prefix): # export PRIVATE_KEY="0x<64-hex-characters>" # or generate one: node dist/helpers/scripts/generatePK.js # if you use a JSON config file instead, point CONFIG_PATH at its absolute path: # export CONFIG_PATH="/absolute/path/to/config.json" -# 4. Start under PM2 +# 3. Start under PM2 pm2 start npm --name "ocean-node" -- run start ``` @@ -121,18 +118,12 @@ own stdout copy in `~/.pm2/logs` with no size cap unless `pm2-logrotate` is inst - **Node.js**: use the version pinned in [`.nvmrc`](.nvmrc) (run `nvm use`). The effective supported runtime is **Node 24+** (`package.json` `engines`), and the `.nvmrc`/Dockerfile - pin the exact tested version (currently 26.12.1). Note that on anything below **22.13** the - node fails to even start, because the SQLite layer uses the built-in `node:sqlite` module - (`ERR_UNKNOWN_BUILTIN_MODULE: node:sqlite`) — but 22.x is not a supported runtime, use 24+. -- **Docker + Docker Compose** (for the metadata database and C2D). + pin the exact tested version (currently 26.12.1). +- **Docker + Docker Compose** (for C2D, and for PostgreSQL or Elasticsearch if you select one + of them; SQLite needs no container). ```bash -# 1. Start a metadata database (Typesense is the default in config.json) -docker-compose -f typesense-compose.yml up -d -# …or Elasticsearch: -# docker-compose -f elasticsearch-compose.yml up -d - -# 2. Install and build +# 1. Install and build nvm use npm install npm run build @@ -151,16 +142,17 @@ prior `npm run build` is required. ## Configuration -- **`PRIVATE_KEY`** (exactly 66 characters: `0x` + 64 hex) is the **only mandatory** - variable. It seeds the node identity (libp2p peerId), the EVM signer and the encryption - keys; everything else has defaults. Generate one with - `node dist/helpers/scripts/generatePK.js`. +- **`PRIVATE_KEY`** (exactly 66 characters: `0x` + 64 hex) and **`DB_TYPE`** are the + **mandatory** variables. `PRIVATE_KEY` seeds the node identity (libp2p peerId), the EVM + signer and the encryption keys; `DB_TYPE` selects the database (`sqlite` needs no other + setting). Everything else has defaults. + Generate a key with `node dist/helpers/scripts/generatePK.js`. - **Config sources**: a JSON config file plus environment-variable overrides. The file is resolved from `CONFIG_PATH`, else `./config.json` in the working directory, and must exist — - the repo ships a ready-to-run [`config.json`](config.json) (Typesense DB, chain + the repo ships a ready-to-run [`config.json`](config.json) (chain `8996`/development, a single Docker C2D environment). Environment variables are merged on top of the file and take precedence. - - ⚠️ **Blank env vars override the file.** Only *unset* variables fall through to + - ⚠️ **Blank env vars override the file.** Only _unset_ variables fall through to `config.json`; an **empty** value (e.g. `export HTTP_API_PORT=`) is treated as set and overrides the file's value. `npm run setupEnv` creates `.env` by copying [`.env.example`](.env.example), which has many blank `export VAR=` lines, so after @@ -170,8 +162,8 @@ prior `npm run build` is required. omitted means both. - [`docs/env.md`](docs/env.md) is the authoritative reference. Key groups worth knowing: - `RPCS` — per-chain RPC endpoint map (JSON). - - `DB_URL` / `DB_TYPE` — metadata DB connection and backend; presence drives whether the - indexer runs. + - `DB_TYPE` — the database, required: `sqlite`, `postgresql` or `elasticsearch`. + `DB_URL` / `DB_USERNAME` / `DB_PASSWORD` — the connection, for the last two. - `ALLOWED_ADMINS`, `AUTHORIZED_DECRYPTERS`, `AUTHORIZED_PUBLISHERS`, `ALLOWED_VALIDATORS` (and their access-list variants) — admin / decrypter / validator allow-lists. - `MAX_REQ_PER_MINUTE`, `RATE_DENY_LIST` — rate limiting. @@ -182,29 +174,29 @@ prior `npm run build` is required. ## Databases -The metadata backend is pluggable: **Typesense** or **Elasticsearch**, chosen via `DB_TYPE`. -Both ship with a compose file ([`typesense-compose.yml`](typesense-compose.yml), -[`elasticsearch-compose.yml`](elasticsearch-compose.yml)) and hold DDOs, indexer state, logs, -orders, access lists and escrow events. - -> **Default differs by config source.** The shipped [`config.json`](config.json) defaults to -> **Typesense**, but when you supply the DB through **environment variables** (`DB_URL`, e.g. -> via `npm run setupEnv`) the builder sets `dbConfig.dbType` to `DB_TYPE || 'elasticsearch'`. -> So if you enable the DB via env vars and want Typesense, you must set `DB_TYPE=typesense` -> explicitly — otherwise the node tries to talk to your Typesense endpoint as Elasticsearch. - -**SQLite is always on**, independent of the metadata DB: it backs the nonce, config, C2D-job -and auth-token stores via Node's built-in `node:sqlite` module (no native addon). A valid -`dbConfig` in `config.json` is **retained** when `DB_URL` is unset or empty (env `DB_TYPE` -alone does not build or override it), so the file's DB config still drives the indexer; it is -disabled outright only when **neither** the file nor the environment provides a valid DB -config (the SQLite stores keep working regardless). A valid `dbConfig` is **necessary but not -sufficient**, though: if the metadata DB is unreachable when the node starts (connection -fails after the init retries), the node logs *DEGRADED mode*, disables the indexer, and keeps -only the SQLite-backed C2D / nonce / auth stores — so also make sure the configured -Typesense/Elasticsearch endpoint is actually up and reachable. A config value of the wrong type instead fails -`OceanNodeConfigSchema` (zod) validation and stops startup before any DB is initialized. See -[Database guide](docs/database.md). +The node keeps all its data in one database, selected with `DB_TYPE`: + +- **`sqlite`**: no database container and no other setting. The files are in the + `databases/` folder. One folder is for one node instance. +- **`postgresql`** or **`elasticsearch`**: for several node instances on one database. Set + `DB_URL`, `DB_USERNAME` and `DB_PASSWORD` (or `dbConfig` in `config.json`); without them the + configuration fails validation and the node does not start. The repo has a compose file for + each: [`postgresql-compose.yml`](postgresql-compose.yml) and + [`elasticsearch-compose.yml`](elasticsearch-compose.yml). + +With no `DB_TYPE` the node does not start: it does not get the type from `DB_URL`. The node +state does not move between databases: a node that changes to a database server starts with +empty node state. See "Upgrade" in the [Database guide](docs/database.md). + +DDO search is the same on all three: a typed query (filters, sort key, page) that each +database runs with its own fixed queries. See "DDO search" in the +[Database guide](docs/database.md#ddo-search). + +There is no DEGRADED mode: if the database cannot be opened when the node starts (after the +init retries), the node logs `Database failed to initialize, exiting` and exits with code 1. +A config value of the wrong type instead fails `OceanNodeConfigSchema` (zod) validation and +stops startup before any DB is initialized. See [Database guide](docs/database.md), also for +the upgrade from an older release. ## Compute (C2D) & services diff --git a/config.json b/config.json index cdc54aa97..70f91b3e1 100644 --- a/config.json +++ b/config.json @@ -59,12 +59,6 @@ "enableNetworkStats": false }, "httpPort": 8001, - "dbConfig": { - "url": "http://localhost:8108/?apiKey=xyz", - "username": "", - "password": "", - "dbType": "typesense" - }, "supportedNetworks": { "8996": { "rpc": "http://127.0.0.1:8545", diff --git a/docs/API.md b/docs/API.md index 08f9249f1..65407f4e1 100644 --- a/docs/API.md +++ b/docs/API.md @@ -34,118 +34,105 @@ signed, so clients that build the message from a lowercase address keep working. ## State DDO -### `HTTP` GET /api/aquarius/assets/metadata/query? +### `HTTP` GET /api/ddo/state #### Description -returns ddo state +Returns the state of a DDO in the indexer: if the node indexed it, and the error if not. #### Query Parameters -| name | type | required | description | -| ------- | ------ | -------- | ------------------------------------------------------ | -| did | object | | document id or did | -| chainId | object | | chain id of network on which document is provided | -| nft | object | | one or more field names that should be queried against | +Give one of the three. + +| name | type | required | description | +| ---- | ------ | -------- | -------------------------------------------------- | +| did | string | | the DID of the document | +| nft | string | | the address of the NFT of the document | +| txId | string | | the id of the transaction that published the event | #### Response -``` -123 +```json +{ + "id": "did:op:fa0e8fa9550e8eb13392d6eeb9ba9f8111801b332c8d2345b350b3bc66b379d5", + "chainId": 137, + "did": "did:op:fa0e8fa9550e8eb13392d6eeb9ba9f8111801b332c8d2345b350b3bc66b379d5", + "nft": "0xBB1081DbF3227bbB233Db68f7117114baBb43656", + "txId": "0x3d7e0c3f...", + "valid": true, + "error": " " +} ``` --- ## Query DDO -### `HTTP` POST /api/aquarius/assets/metadata/query +### `HTTP` POST /api/query #### Description -returns search result for query +Searches the DDOs, the orders of an account, or the tags of the DDOs. The query is the same on +all databases: a type, typed filters, a sort key and a page (see +[database.md](database.md#ddo-search) for each filter). An Elasticsearch query body gives +HTTP 400. #### Parameters -| name | type | required | description | -| -------- | ------ | -------- | ------------------------------------------------------ | -| q | object | v | text to search for in database | -| query_by | object | v | one or more field names that should be queried against | +| name | type | required | description | +| ------ | ------ | -------- | --------------------------------------------------------------------------------------------------------------------- | +| type | string | | `assets` (the default), `orders` or `tags` | +| filter | object | | the asset filters; with `orders`: `{ "consumer": … }` or `{ "payer": … }` | +| sort | object | | `assets` only: `{ "by": "created" \| "orders" \| "price" \| "tokenSymbol" \| "relevance", "order": "asc" \| "desc" }` | +| page | number | | page number, from 1 | +| size | number | | documents for each page, 1000 when absent; with `tags`: the number of tags | #### Request ```json { - "q": "0x123", - "query_by": "nftAddress" -} -``` - -#### Response - -``` -[{ - "facet_counts": [], - "found": 1, - "out_of": 1, - "page": 1, - "request_params": { - "collection_name": "ddo", - "per_page": 10, - "q": "0x123" + "filter": { + "chainIds": [1, 137], + "types": ["dataset"], + "serviceTypes": ["access"], + "text": "weather" }, - "search_time_ms": 1, - "hits": [ - { - "highlights": [ - { - "field": "nftAddress", - "snippet": "0x123", - "matched_tokens": ["0x123"] - } - ], - "document": { - "@context": ["https://w3id.org/did/v1"], - "id": "did:op:ACce67694eD2848dd683c651Dab7Af823b7dd123", - "version": "4.1.0", - "chainId": 1, - "nftAddress": "0x123", - ... - }, - "text_match": 130916 - } - ] -}] + "sort": { "by": "created", "order": "desc" }, + "page": 1, + "size": 10 +} ``` ---- - -## Get DDO Metadata - -### `HTTP` GET /api/aquarius/assets/ddo/:did - -#### Description - -returns metadata of document by id +`page` × `size` is 10000 at most. #### Response -``` +One list for the DDOs of all versions, sorted and paged over all of them, and the number of +all matches. + +```json { - "created": "2020-11-15T12:27:48Z", - "updated": "2021-05-17T21:58:02Z", - "description": "Sample description", - "name": "Sample asset", - "type": "dataset", - "author": "OPF", - "license": "https://market.oceanprotocol.com/terms" + "results": [ + { + "id": "did:op:fa0e8fa9550e8eb13392d6eeb9ba9f8111801b332c8d2345b350b3bc66b379d5", + "chainId": 137, + "nftAddress": "0xBB1081DbF3227bbB233Db68f7117114baBb43656", + "version": "4.1.0", + "metadata": { "name": "dataset-name", "type": "dataset" } + } + ], + "totalResults": 1 } ``` +With `type: "tags"`, `results` holds `{ "tag", "count" }` objects. A query that is not valid +gives HTTP 400 with the reason as text, for example `Invalid query: filter.types: …`. + --- ## Get DDO -### `HTTP` GET /api/aquarius/assets/ddo/:did +### `HTTP` GET /api/ddo/:did #### Description @@ -303,17 +290,17 @@ returns decrypted document #### Parameters -| name | type | required | description | -| ----------------- | ------ | -------- | --------------------------------------------------------------------------------------------- | -| decrypterAddress | string | v | decrypter address | -| chainId | number | v | chain id of network on which document is provided | -| transactionId | string | | tx id of encrypted document | -| dataNftAddress | string | | address of nft token | -| encryptedDocument | string | | encrypted document | -| flags | number | | metadata flags if DDO is ECIES encrypted or lzma compressed | -| documentHash | string | | hash based on sha256 of asset | -| nonce | string | v | is required to verify a request paired with a signature | -| signature | string | v | signed message (see below) | +| name | type | required | description | +| ----------------- | ------ | -------- | ----------------------------------------------------------- | +| decrypterAddress | string | v | decrypter address | +| chainId | number | v | chain id of network on which document is provided | +| transactionId | string | | tx id of encrypted document | +| dataNftAddress | string | | address of nft token | +| encryptedDocument | string | | encrypted document | +| flags | number | | metadata flags if DDO is ECIES encrypted or lzma compressed | +| documentHash | string | | hash based on sha256 of asset | +| nonce | string | v | is required to verify a request paired with a signature | +| signature | string | v | signed message (see below) | > **ON-02 signed-message format (BREAKING, since Ocean Node v4.8.0).** The `decryptDDO` > signature now binds the **destination node** and the **exact resource**, so a signature cannot @@ -376,14 +363,14 @@ Files whose host is, or resolves to, a private, loopback, link-local or otherwis #### Parameters -| name | type | required | description | -| --------- | ------ | -------- | --------------------------------------------------- | -| type | string | | type of storage `url or arweave or ipfs` | -| did | string | | document id or did | -| serviceId | number | | service id of services list | -| fileIndex | number | | file index in files array | -| file | object | | file data | -| checksum | number | | index in transaction events list | +| name | type | required | description | +| --------- | ------ | -------- | ---------------------------------------- | +| type | string | | type of storage `url or arweave or ipfs` | +| did | string | | document id or did | +| serviceId | number | | service id of services list | +| fileIndex | number | | file index in files array | +| file | object | | file data | +| checksum | number | | index in transaction events list | #### Raw-`file` path hardening (ON-07) @@ -472,12 +459,12 @@ returns log by id. Admin only. Works only with `LOG_DB` and an available log dat #### Parameters -| name | type | required | description | -| --------- | ------ | -------- | ----------------------------------------------------- | -| logId | string | v | id of the log entry (read from the body) | -| address | string | v | admin address (must be in `ALLOWED_ADMINS`) | +| name | type | required | description | +| --------- | ------ | -------- | ---------------------------------------------------------------- | +| logId | string | v | id of the log entry (read from the body) | +| address | string | v | admin address (must be in `ALLOWED_ADMINS`) | | signature | string | v | signed message (auth tokens are not accepted for admin commands) | -| nonce | string | v | nonce used for the signature | +| nonce | string | v | nonce used for the signature | #### Response @@ -682,34 +669,39 @@ returns list of all P2P peers ## Validate DDO -### `HTTP` POST /directCommand +### `HTTP` POST /api/ddo/validate ### `P2P` command: validateDDO #### Description -returns an empty object if it is valid otherwise an array with error +Checks a DDO against its schema (and the policy server, when one is configured). A valid DDO +gets the signed hash of the node, which the publisher writes on chain with the metadata. With +`VALIDATE_UNSIGNED_DDO=false`, only a request with a signature or an auth token gets the +signed hash; an unsigned request gets `200` with an empty body. #### Parameters -| name | type | required | description | -| ---------- | -------- | -------- | ------------------------------------------------- | -| command | string | v | command name | -| node | string | | if not present it means current node | -| multiAddrs | string[] | | if passed, use this instead of peerStore & DHT | -| id | string | v | document id or did | -| chainId | number | v | chain id of network on which document is provided | -| nftAddress | string | v | address of nft token | +| name | type | required | description | +| ---------------- | ------ | -------- | ------------------------------------------------------- | +| ddo | object | v | the DDO | +| publisherAddress | string | | the address that signs | +| nonce | string | | the next nonce of the publisher | +| signature | string | | signature of `publisherAddress + nonce + "validateDDO"` | +| Authorization | header | | an auth token, in place of the signature (HTTP) | -#### Request +#### Response + +`200` with the signed hash. `400` with the validation error. `403` when the policy server +denies the publisher. ```json { - "command": "validateDDO", - "node": "PeerId", - "id": "did:op:123", - "chainId": 123, - "nftAddress": "0x123" + "hash": "0x…", + "publicKey": "0x…", + "r": ["0x…"], + "s": ["0x…"], + "v": 27 } ``` @@ -1019,17 +1011,17 @@ empty array is always allowed). Free compute has no escrow lock/claim, so the fi #### Description -returns search result for query +Searches the DDOs, the orders of an account, or the tags of the DDOs, with the same query as +`POST /api/query` (see [database.md](database.md#ddo-search)). An +Elasticsearch query body gives HTTP 400. #### Parameters -| name | type | required | description | -| -------------- | ------ | -------- | ------------------------------------------------------ | -| command | string | v | command name | -| node | string | | if not present it means current node | -| query | object | v | query parameters | -| query.q | object | v | text to search for in database | -| query.query_by | object | v | one or more field names that should be queried against | +| name | type | required | description | +| ------- | ------ | -------- | --------------------------------------------------- | +| command | string | v | command name | +| node | string | | if not present it means current node | +| query | object | v | the query: `type`, `filter`, `sort`, `page`, `size` | #### Request @@ -1038,48 +1030,16 @@ returns search result for query "command": "query", "node": "PeerId", "query": { - "q": "0x123", - "query_by": "nftAddress" + "type": "orders", + "filter": { "consumer": "0x1234567890123456789012345678901234567890" }, + "size": 10 } } ``` #### Response -``` -[{ - "facet_counts": [], - "found": 1, - "out_of": 1, - "page": 1, - "request_params": { - "collection_name": "ddo", - "per_page": 10, - "q": "0x123" - }, - "search_time_ms": 1, - "hits": [ - { - "highlights": [ - { - "field": "nftAddress", - "snippet": "0x123", - "matched_tokens": ["0x123"] - } - ], - "document": { - "@context": ["https://w3id.org/did/v1"], - "id": "did:op:ACce67694eD2848dd683c651Dab7Af823b7dd123", - "version": "4.1.0", - "chainId": 1, - "nftAddress": "0x123", - ... - }, - "text_match": 130916 - } - ] -}] -``` +`{ "results": [...], "totalResults": n }`, as the HTTP route answers. --- @@ -1214,19 +1174,19 @@ returns decrypted document #### Parameters -| name | type | required | description | -| ----------------- | ------ | -------- | --------------------------------------------------------------------------------------------- | -| command | string | v | command name | -| node | string | | if not present it means current node | -| decrypterAddress | string | v | decrypter address | -| chainId | number | v | chain id of network on which document is provided | -| transactionId | string | | tx id of encrypted document | -| dataNftAddress | string | | address of nft token | -| encryptedDocument | string | | encrypted document | -| flags | number | | metadata flags if DDO is ECIES encrypted or lzma compressed | -| documentHash | string | | hash based on sha256 of asset | -| nonce | string | v | is required to verify a request paired with a signature | -| signature | string | v | signed message (see below) | +| name | type | required | description | +| ----------------- | ------ | -------- | ----------------------------------------------------------- | +| command | string | v | command name | +| node | string | | if not present it means current node | +| decrypterAddress | string | v | decrypter address | +| chainId | number | v | chain id of network on which document is provided | +| transactionId | string | | tx id of encrypted document | +| dataNftAddress | string | | address of nft token | +| encryptedDocument | string | | encrypted document | +| flags | number | | metadata flags if DDO is ECIES encrypted or lzma compressed | +| documentHash | string | | hash based on sha256 of asset | +| nonce | string | v | is required to verify a request paired with a signature | +| signature | string | v | signed message (see below) | > **ON-02 signed-message format (BREAKING, since Ocean Node v4.8.0).** Same bound message as the > HTTP `/api/services/decrypt` endpoint: @@ -1548,7 +1508,7 @@ process kill). While it is ON, the node: - **Everything else is unchanged**: download, DDO query/validate, indexer, P2P, admin, fees, nonce, persistent storage, access lists, escrow events, auth tokens. -The flag is persisted in the SQLite config DB, so it is **sticky across restarts**: a node that +The flag is persisted in the database of the node, so it is **sticky across restarts**: a node that reboots while in maintenance comes back up still in maintenance and will not accept new work until an admin disables it. It can also be set at boot with the `MAINTENANCE_MODE` env var (see `docs/env.md`). There is **no TTL and no auto-exit**: reaching zero running jobs does nothing on @@ -1561,7 +1521,10 @@ Rejected requests return HTTP **`503 Service Unavailable`** with a stable machin in the JSON response body: ```json -{ "error": "Node is in maintenance mode; not accepting new compute or service requests", "code": "MAINTENANCE_MODE" } +{ + "error": "Node is in maintenance mode; not accepting new compute or service requests", + "code": "MAINTENANCE_MODE" +} ``` > **Important:** `503` is also returned by these handlers for other reasons (e.g. @@ -1594,13 +1557,13 @@ Toggles maintenance mode (admin only). Signed with the standard admin fields and #### Parameters -| name | type | required | description | -| --------- | ------- | -------- | ------------------------------------------------ | -| command | string | POST /directCommand | command name (`setMaintenance`) | -| enabled | boolean | v | `true` enables maintenance, `false` disables it | -| nonce | string | v | request nonce | -| address | string | v | admin address | -| signature | string | v | signed message to authenticate the admin request | +| name | type | required | description | +| --------- | ------- | ------------------- | ------------------------------------------------ | +| command | string | POST /directCommand | command name (`setMaintenance`) | +| enabled | boolean | v | `true` enables maintenance, `false` disables it | +| nonce | string | v | request nonce | +| address | string | v | admin address | +| signature | string | v | signed message to authenticate the admin request | #### Request (POST /api/admin/maintenance) @@ -1649,11 +1612,11 @@ Both `status` and `detailedStatus` expose a `maintenance` object. `enabled` is a } ``` -| field | type | status level | description | -| ----------------------- | ------- | ------------ | ----------------------------------------------------- | -| `enabled` | boolean | always | whether the node is in maintenance mode | -| `runningComputeJobs` | number | detailed | compute jobs still holding resources | -| `runningServices` | number | detailed | service-on-demand jobs still running | +| field | type | status level | description | +| ----------------------- | ------- | ------------ | ------------------------------------------------------ | +| `enabled` | boolean | always | whether the node is in maintenance mode | +| `runningComputeJobs` | number | detailed | compute jobs still holding resources | +| `runningServices` | number | detailed | service-on-demand jobs still running | | `latestExpectedEndTime` | number | detailed | max job/service end time (unix seconds), the drain ETA | --- @@ -1672,22 +1635,22 @@ Returns indexed Escrow contract events. The indexer matches Escrow logs by topic #### Parameters -| name | type | required | description | -| ----------- | ------- | --------- | --------------------------------------------------------- | -| command | string | POST only | command name (`getEscrowEvents`) | -| chainId | number | | chain id | +| name | type | required | description | +| ----------- | ------- | --------- | ------------------------------------------------------------------------------------------------------------- | +| command | string | POST only | command name (`getEscrowEvents`) | +| chainId | number | | chain id | | eventType | string | | one of `Auth, Lock, ReLock, Claimed, Canceled, Deposit, Withdraw, Subsidized, LockSponsored, SponsorRefunded` | -| payer | string | | payer address (case-insensitive) | -| payee | string | | payee address (case-insensitive) | -| token | string | | token address (case-insensitive) | -| provider | string | | subsidy/sponsor provider address (case-insensitive) | -| contract | string | | Escrow contract address | -| reclaimable | boolean | | filter `SponsorRefunded` rows by their reclaimable flag | -| jobId | string | | compute job id | -| txId | string | | transaction hash | -| ranges | object | POST only | range filters keyed by field (see below) | -| offset | number | | rows to skip (default 0) | -| size | number | | page size (default 100, max 250) | +| payer | string | | payer address (case-insensitive) | +| payee | string | | payee address (case-insensitive) | +| token | string | | token address (case-insensitive) | +| provider | string | | subsidy/sponsor provider address (case-insensitive) | +| contract | string | | Escrow contract address | +| reclaimable | boolean | | filter `SponsorRefunded` rows by their reclaimable flag | +| jobId | string | | compute job id | +| txId | string | | transaction hash | +| ranges | object | POST only | range filters keyed by field (see below) | +| offset | number | | rows to skip (default 0) | +| size | number | | page size (default 100, max 250) | No authentication is required (both the HTTP route and the P2P command are public reads). @@ -1703,9 +1666,9 @@ Any field above is an exact-match filter. Numeric fields also support range filt Range-filterable fields: -| field | pushed to store? | -| ----------------------------------------------------------------------------------------- | ---------------- | -| `block`, `chainId` | yes (native numeric) | +| field | pushed to store? | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `block`, `chainId` | yes (native numeric) | | `amount`, `subsidyAmount`, `bonusAmount`, `oldAmount`, `newAmount`, `maxLockedAmount`, `maxLockSeconds`, `maxLockCounts`, `expiry`, `newExpiry`, `expiryTimestamp` | no — filtered node-side with BigInt comparison over paginated store results (bounded scan; a highly selective amount range over a very large result set may be truncated) | A range on any field not in this list (e.g. an address field) is rejected with `400`. @@ -1823,17 +1786,17 @@ Returns a live per-node resource snapshot, rolled up across every C2D engine (th #### Description -Returns ordered hourly averages of the per-node resource snapshot, persisted to SQLite by the sampler/roll-up cron jobs and retained for `NODE_METRICS_RETENTION_DAYS` (default 180). Scalars are arithmetic means over the hour's minute-samples; `sampleCount` is how many samples fed each bucket; GPU entries are averaged per `resourceId`, env entries per `env`+`resource`. Requires the node-metrics database (returns `503` when unavailable, e.g. history disabled via `NODE_METRICS_HISTORY_ENABLED=false`). +Returns ordered hourly averages of the per-node resource snapshot, persisted to the database of the node per instance (`INSTANCE_ID`) by the sampler/roll-up cron jobs and retained for `NODE_METRICS_RETENTION_DAYS` (default 180). Scalars are arithmetic means over the hour's minute-samples; `sampleCount` is how many samples fed each bucket; GPU entries are averaged per `resourceId`, env entries per `env`+`resource`. Requires the node-metrics database (returns `503` when unavailable, e.g. history disabled via `NODE_METRICS_HISTORY_ENABLED=false`). When the requested range includes the current, in-progress hour, the last bucket is a **live** average computed on the fly from the raw samples collected so far this hour (before the top-of-hour roll-up has stored it). It is flagged `"partial": true` and is the only bucket that carries that flag; every completed hour is a finalized, stored average. This lets a caller see fresh data without waiting for the hourly roll-up. #### Parameters -| name | type | required | description | -| --------- | ------------- | -------- | ---------------------------------------------------------------------------------- | -| command | string | POST only | command name (`getNodeMetricsHistory`) | -| startTime | number/string | | range start — epoch ms or ISO-8601. Defaults to now minus the retention window | -| stopTime | number/string | | range end — epoch ms or ISO-8601. Defaults to now | +| name | type | required | description | +| --------- | ------------- | --------- | ------------------------------------------------------------------------------ | +| command | string | POST only | command name (`getNodeMetricsHistory`) | +| startTime | number/string | | range start — epoch ms or ISO-8601. Defaults to now minus the retention window | +| stopTime | number/string | | range end — epoch ms or ISO-8601. Defaults to now | `startTime` must be earlier than `stopTime` (else `400`); the range is clamped to the retention window and the row count is capped. @@ -1858,7 +1821,12 @@ When the requested range includes the current, in-progress hour, the last bucket { "hourStart": 1730368800000, "sampleCount": 60, - "cpu": { "usagePercent": 40.1, "coresAllocated": 4, "hostCores": 16, "throttledCount": 0 }, + "cpu": { + "usagePercent": 40.1, + "coresAllocated": 4, + "hostCores": 16, + "throttledCount": 0 + }, "memory": { "usedBytes": 2000000000, "limitBytes": 8589934592, @@ -1997,7 +1965,7 @@ and SERVICE_START rejects a `duration` outside it: value is clamped to the daemon ceiling at startup. SERVICE_EXTEND also caps the resulting remaining window to it. - `minServiceDuration` — the floor. SERVICE_START rejects a shorter `duration`, and - SERVICE_EXTEND rejects a shorter `additionalDuration`: the floor is a minimum *purchase*, so a + SERVICE_EXTEND rejects a shorter `additionalDuration`: the floor is a minimum _purchase_, so a smaller one is refused rather than silently billed at the floor. Everything accepted is then priced by its actual duration, rounded up to whole minutes. Defaults to the environment's own `minJobDuration`, and may be **raised** per environment; a value below the daemon's @@ -2099,14 +2067,14 @@ returns job status Required at least one of the following parameters: -| name | type | required | description | -| --------------- | ------- | -------- | ------------------------------------------------------------------------------------------------------------ | -| consumerAddress | string | | consumer address to use as filter | -| jobId | string | | jobId address to use as filter | -| agreementId | string | | agreementId address to use as filter | -| includeMetrics | boolean | | override the runtime-metrics default (`true` = require them, `false` = never). See note below. | +| name | type | required | description | +| --------------- | ------- | -------- | --------------------------------------------------------------------------------------------------- | +| consumerAddress | string | | consumer address to use as filter | +| jobId | string | | jobId address to use as filter | +| agreementId | string | | agreementId address to use as filter | +| includeMetrics | boolean | | override the runtime-metrics default (`true` = require them, `false` = never). See note below. | | signature | string | | signature over `consumerAddress` + `nonce` + `command` (or an auth token) — authenticates the owner | -| nonce | string | | request nonce, paired with `signature` | +| nonce | string | | request nonce, paired with `signature` | **Runtime metrics are owner-only and returned BY DEFAULT.** If the request carries owner credentials (`consumerAddress` plus `signature`/`nonce`, or an `Authorization` header token), each job owned by @@ -2114,11 +2082,11 @@ Required at least one of the following parameters: `includeMetrics` only overrides that default: -| `includeMetrics` | behavior | -| --- | --- | +| `includeMetrics` | behavior | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | omitted (default) | Metrics attached when owner credentials are present and valid. A request without credentials — the plain, unauthenticated status call — returns `200` with no metrics, exactly as before. Invalid credentials likewise just mean no metrics. | -| `true` | Metrics are **required**: missing `consumerAddress` answers `400`, failed authentication `401`. Use it when you want to know *why* metrics are absent instead of getting a silently trimmed response. | -| `false` | Metrics are never attached (and the node skips the auth round-trip). | +| `true` | Metrics are **required**: missing `consumerAddress` answers `400`, failed authentication `401`. Use it when you want to know _why_ metrics are absent instead of getting a silently trimmed response. | +| `false` | Metrics are never attached (and the node skips the auth round-trip). | Metrics never reach a non-owner, and are never part of the on-chain escrow claim proof. They are best-effort and up to one sampling interval stale (see [compute.md](compute.md) and @@ -2190,71 +2158,71 @@ Clients MUST treat every part as optional and render a field only when present. #### Top-level fields -| field | type | unit / notes | -| --------------- | ------- | ------------------------------------------------------------------------------------------------ | -| collectedAt | string | ISO-8601 timestamp of the sample | -| containerState | object | see below — status + structured exit info | -| cpu | object | see below | -| memory | object | see below | -| disk | object | see below | -| network | object? | `{ rxBytes, txBytes }`; **absent** when the container runs with no network (`NetworkMode: none`) | -| blockIO | object | `{ readBytes, writeBytes }` — cumulative disk I/O in bytes | -| pids | object | `{ current, limit }` — process/thread count vs the container PID limit (512) | -| gpu | array? | one entry per GPU the job/service holds; **absent** for CPU-only jobs or when GPU metrics are off | +| field | type | unit / notes | +| -------------- | ------- | ------------------------------------------------------------------------------------------------- | +| collectedAt | string | ISO-8601 timestamp of the sample | +| containerState | object | see below — status + structured exit info | +| cpu | object | see below | +| memory | object | see below | +| disk | object | see below | +| network | object? | `{ rxBytes, txBytes }`; **absent** when the container runs with no network (`NetworkMode: none`) | +| blockIO | object | `{ readBytes, writeBytes }` — cumulative disk I/O in bytes | +| pids | object | `{ current, limit }` — process/thread count vs the container PID limit (512) | +| gpu | array? | one entry per GPU the job/service holds; **absent** for CPU-only jobs or when GPU metrics are off | `containerState`: -| field | type | notes | -| ------------ | -------- | ------------------------------------------------------------------------ | -| status | string | e.g. `running`, `exited` | -| startedAt | string? | ISO-8601 | -| finishedAt | string? | ISO-8601; present once the container has stopped | -| exitCode | number? | process exit code (present after exit) | -| oomKilled | boolean | `true` if the kernel OOM-killed the container | -| error | string? | Docker-reported error string, if any | -| restartCount | number | container restarts | -| health | string? | Docker HEALTHCHECK status when the image defines one (e.g. `healthy`) | +| field | type | notes | +| ------------ | ------- | --------------------------------------------------------------------- | +| status | string | e.g. `running`, `exited` | +| startedAt | string? | ISO-8601 | +| finishedAt | string? | ISO-8601; present once the container has stopped | +| exitCode | number? | process exit code (present after exit) | +| oomKilled | boolean | `true` if the kernel OOM-killed the container | +| error | string? | Docker-reported error string, if any | +| restartCount | number | container restarts | +| health | string? | Docker HEALTHCHECK status when the image defines one (e.g. `healthy`) | `cpu`: -| field | type | unit / notes | -| ---------------------- | ------ | ------------------------------------------------------------------------------- | -| usagePercent | number | % of one host CPU-second per wall-second (docker-stats formula), `0–N×100` | -| allocated | number | CPU cores requested by the job (`0` when unconstrained) | -| usagePercentOfAllocated| number | `usagePercent / allocated` — "how saturated is what you paid for" (`0` if alloc 0)| -| cumulativeSeconds | number | total CPU-seconds consumed since start (monotonic; billing-grade) | -| throttledPeriods | number | CFS quota throttling events — high ⇒ the CPU request is too small | -| throttledSeconds | number | total time throttled, seconds | +| field | type | unit / notes | +| ----------------------- | ------ | ---------------------------------------------------------------------------------- | +| usagePercent | number | % of one host CPU-second per wall-second (docker-stats formula), `0–N×100` | +| allocated | number | CPU cores requested by the job (`0` when unconstrained) | +| usagePercentOfAllocated | number | `usagePercent / allocated` — "how saturated is what you paid for" (`0` if alloc 0) | +| cumulativeSeconds | number | total CPU-seconds consumed since start (monotonic; billing-grade) | +| throttledPeriods | number | CFS quota throttling events — high ⇒ the CPU request is too small | +| throttledSeconds | number | total time throttled, seconds | `memory`: -| field | type | unit / notes | -| -------------- | ------ | -------------------------------------------------------- | -| usageBytes | number | working-set bytes (`usage − inactive_file`, cgroup v2) | -| limitBytes | number | memory limit (= allocated RAM) in bytes | -| usagePercent | number | `usageBytes / limitBytes × 100` | -| peakUsageBytes | number | max `usageBytes` observed across samples | +| field | type | unit / notes | +| -------------- | ------ | ------------------------------------------------------ | +| usageBytes | number | working-set bytes (`usage − inactive_file`, cgroup v2) | +| limitBytes | number | memory limit (= allocated RAM) in bytes | +| usagePercent | number | `usageBytes / limitBytes × 100` | +| peakUsageBytes | number | max `usageBytes` observed across samples | `disk`: -| field | type | unit / notes | -| ------------ | ------- | ---------------------------------------------------------------------------------- | +| field | type | unit / notes | +| ------------ | ------- | ------------------------------------------------------------------------------------- | | usedBytes | number | compute jobs: bytes written under `/` (excludes base image); services: writable layer | -| quotaBytes | number? | present only for jobs with a `disk` resource | -| usagePercent | number? | present only when `quotaBytes` is known | +| quotaBytes | number? | present only for jobs with a `disk` resource | +| usagePercent | number? | present only when `quotaBytes` is known | `gpu[]` entry: -| field | type | unit / notes | -| ------------------ | --------------- | --------------------------------------------------------------------- | -| resourceId | string | the requested resource id (`gpu0`, `gpu1`, …) — maps the entry to a device | -| vendor | string | `nvidia` (only NVIDIA is emitted today; `amd`/`intel` reserved) | -| utilizationPercent | number \| null | GPU busy % (`null` = unreadable) | -| memoryUsedBytes | number \| null | VRAM used | -| memoryTotalBytes | number \| null | total VRAM | -| temperatureC | number? | °C, when available | -| powerWatts | number? | current draw, W, when available | -| shared | boolean? | `true` ⇒ device is shareable and the number may include other jobs | +| field | type | unit / notes | +| ------------------ | -------------- | -------------------------------------------------------------------------- | +| resourceId | string | the requested resource id (`gpu0`, `gpu1`, …) — maps the entry to a device | +| vendor | string | `nvidia` (only NVIDIA is emitted today; `amd`/`intel` reserved) | +| utilizationPercent | number \| null | GPU busy % (`null` = unreadable) | +| memoryUsedBytes | number \| null | VRAM used | +| memoryTotalBytes | number \| null | total VRAM | +| temperatureC | number? | °C, when available | +| powerWatts | number? | current draw, W, when available | +| shared | boolean? | `true` ⇒ device is shareable and the number may include other jobs | > Note: the node also keeps an internal delta accumulator on the stored snapshot; it is stripped > from the response and clients will never see it. @@ -2484,10 +2452,10 @@ Raw file bytes, sent with: #### Errors -| status | when | +| status | when | | ------ | ---------------------------------------------------------- | | 403 | consumer is not the bucket owner and not on the bucket ACL | -| 404 | file not found in the bucket | +| 404 | file not found in the bucket | --- @@ -2609,20 +2577,20 @@ must pass that key's `validation`, and the key may not also be an operator `envV The encrypted `userData` is never returned. Key fields: -| property | type | description | -| -------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| serviceId | string | unique id of the running service | -| environment | string | envId the service runs on | -| owner | string | consumerAddress | +| property | type | description | +| -------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| serviceId | string | unique id of the running service | +| environment | string | envId the service runs on | +| owner | string | consumerAddress | | status | number | `10` Starting, `20` Locking, `11` PullImage, `13` BuildImage, `30` Claiming, `40` Running, `12` PullImageFailed, `14` BuildImageFailed, `15` VulnerableImage, `50` Stopping, `70` Stopped, `75` Expired, `76` DiskQuotaExceeded (restartable/editable like Stopped until expiresAt), `99` Error | -| statusText | string | human-readable status | -| dateCreated | string | ISO timestamp | -| expiresAt | number | Unix ms timestamp when the paid window ends | -| duration | number | requested seconds | -| endpoints | object[] | `{ containerPort, hostPort, url }` per exposed port | -| resources | object[] | `{ id, amount, price }` | -| payment | object | initial start payment record | -| extendPayments | object[] | one entry per successful extend | +| statusText | string | human-readable status | +| dateCreated | string | ISO timestamp | +| expiresAt | number | Unix ms timestamp when the paid window ends | +| duration | number | requested seconds | +| endpoints | object[] | `{ containerPort, hostPort, url }` per exposed port | +| resources | object[] | `{ id, amount, price }` | +| payment | object | initial start payment record | +| extendPayments | object[] | one entry per successful extend | --- @@ -2703,19 +2671,19 @@ should keep watching `serviceStatus`, not just stop once they first see `Running } ``` -| field | type | required | description | -| ---------------------------- | -------- | -------- | ----------------------------------------------------------------- | -| environment | string | v | envId to run on (services must be enabled on it) | -| image | string | v | base image | -| tag / checksum / dockerfile | string | | image spec — at most one; `dockerfile` requires `allowImageBuild` | -| additionalDockerFiles | object | | filename → content; only with `dockerfile` | -| dockerCmd / dockerEntrypoint | string[] | | container CMD / ENTRYPOINT overrides | -| exposedPorts | number[] | | container ports to publish | -| resources | object[] | | `{ id, amount }` requested resources | -| duration | number | v | seconds; capped by `serviceOnDemand.maxDurationSeconds` | -| userData | string | | ECIES-encrypted (to the node pubkey) JSON of env vars | +| field | type | required | description | +| ---------------------------- | -------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| environment | string | v | envId to run on (services must be enabled on it) | +| image | string | v | base image | +| tag / checksum / dockerfile | string | | image spec — at most one; `dockerfile` requires `allowImageBuild` | +| additionalDockerFiles | object | | filename → content; only with `dockerfile` | +| dockerCmd / dockerEntrypoint | string[] | | container CMD / ENTRYPOINT overrides | +| exposedPorts | number[] | | container ports to publish | +| resources | object[] | | `{ id, amount }` requested resources | +| duration | number | v | seconds; capped by `serviceOnDemand.maxDurationSeconds` | +| userData | string | | ECIES-encrypted (to the node pubkey) JSON of env vars | | metadata | object | | arbitrary user labels (`string`/`number`/`boolean` values, ≤1 KB JSON); node-opaque. Returned on both `serviceStatus` and `serviceList` | -| payment | object | v | `{ chainId, token }` | +| payment | object | v | `{ chainId, token }` | #### Response (200) @@ -2756,12 +2724,12 @@ by the authenticated `consumerAddress` are returned. #### Query Parameters -| name | type | required | description | -| --------------- | ------- | -------- | ----------- | -| consumerAddress | string | v | owner address | -| nonce | string | v | request nonce | -| signature | string | v | signed message (or use an `Authorization` auth-token header) | -| serviceId | string | | filter to a single service; omit to list all owned services | +| name | type | required | description | +| --------------- | ------- | -------- | ------------------------------------------------------------------------------------- | +| consumerAddress | string | v | owner address | +| nonce | string | v | request nonce | +| signature | string | v | signed message (or use an `Authorization` auth-token header) | +| serviceId | string | | filter to a single service; omit to list all owned services | | includeMetrics | boolean | | runtime metrics (`runtimeMetrics`) are included by default; pass `false` to omit them | #### Response (200) @@ -2899,17 +2867,17 @@ RESPEC mode (restart on a new image spec — `image` required, plus at most one } ``` -| name | type | required | description | -| --------------------- | -------- | -------- | ------------------------------------------------------------------------------------------------------------------------ | -| serviceId | string | v | the service to restart | -| image | string | RESPEC | base image name (build label when `dockerfile` is set). Required as soon as any container param is present | -| tag | string | | pull by `name:tag`; mutually exclusive with `checksum`/`dockerfile` | -| checksum | string | | pull by digest `sha256:<64 hex>`; mutually exclusive with `tag`/`dockerfile` | -| dockerfile | string | | build from an inline Dockerfile; requires `allowImageBuild` on the environment; mutually exclusive with `tag`/`checksum` | -| additionalDockerFiles | object | | extra `filename → content` files for the build context (only with `dockerfile`) | -| userData | string | | ECIES-encrypted (to the node public key) JSON → the container's env-var map | -| dockerCmd | string[] | | exact container command (Docker exec-form CMD override) | -| dockerEntrypoint | string[] | | container ENTRYPOINT override | +| name | type | required | description | +| --------------------- | -------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| serviceId | string | v | the service to restart | +| image | string | RESPEC | base image name (build label when `dockerfile` is set). Required as soon as any container param is present | +| tag | string | | pull by `name:tag`; mutually exclusive with `checksum`/`dockerfile` | +| checksum | string | | pull by digest `sha256:<64 hex>`; mutually exclusive with `tag`/`dockerfile` | +| dockerfile | string | | build from an inline Dockerfile; requires `allowImageBuild` on the environment; mutually exclusive with `tag`/`checksum` | +| additionalDockerFiles | object | | extra `filename → content` files for the build context (only with `dockerfile`) | +| userData | string | | ECIES-encrypted (to the node public key) JSON → the container's env-var map | +| dockerCmd | string[] | | exact container command (Docker exec-form CMD override) | +| dockerEntrypoint | string[] | | container ENTRYPOINT override | | metadata | object | | user labels (≤1 KB JSON). **Not** a container param — independent of REUSE/RESPEC. When present it **replaces** the stored metadata; when omitted the original metadata is kept | #### Response (200) @@ -3015,14 +2983,14 @@ container's folder (`live=true`). #### Query Parameters -| name | type | required | description | -| --------------- | ------- | -------- | ---------------------------------------------------------------------------- | -| consumerAddress | string | v | owner address | -| nonce | string | v | request nonce | -| signature | string | v | signed message (or use an `Authorization` auth-token header) | -| serviceId | string | v | the service | -| index | number | | archive index from `outputArchives`; required unless `live=true` | -| offset | number | | resume an archive download from this byte (not with `live`) | +| name | type | required | description | +| --------------- | ------- | -------- | --------------------------------------------------------------------------- | +| consumerAddress | string | v | owner address | +| nonce | string | v | request nonce | +| signature | string | v | signed message (or use an `Authorization` auth-token header) | +| serviceId | string | v | the service | +| index | number | | archive index from `outputArchives`; required unless `live=true` | +| offset | number | | resume an archive download from this byte (not with `live`) | | live | boolean | | `true` to zip the running container's `/data/outputs` instead of an archive | #### Response (200) diff --git a/docs/Arhitecture.md b/docs/Arhitecture.md index 2e2690183..2f1e6bf3b 100644 --- a/docs/Arhitecture.md +++ b/docs/Arhitecture.md @@ -91,7 +91,7 @@ flowchart TB - Node identity is derived from a single `PRIVATE_KEY`: it seeds the libp2p peerId, the EVM signer, and the encryption keys ([libp2p supports ECDSA key pairs](https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#ecdsa)). - Each component can be enabled/disabled at startup (e.g. run with `INTERFACES=["HTTP"]` - only, or without the Indexer when no metadata DB is configured). + only). - C2D: only the Docker engine (`C2DEngineDocker`) is implemented. OPF_K8 clusters configured via `OPERATOR_SERVICE_URL` are accepted into the config (added to `c2dClusters`), but no engine is instantiated for them. @@ -176,14 +176,14 @@ flowchart LR ON --> P2P["OceanP2P
libp2p node"] subgraph DBLAYER["Database"] - META["Metadata DB
Typesense OR Elasticsearch"] - SQL["SQLite (node:sqlite)
nonce · config · C2D jobs · auth tokens"] + META["Index data
SQLite · PostgreSQL · Elasticsearch"] + STATE["Node state
the same database
nonce · config · C2D jobs · auth tokens"] end ON --> DBLAYER IDX --> META PROV --> META - C2D --> SQL - AUTH --> SQL + C2D --> STATE + AUTH --> STATE ``` ### 3.1 Indexer @@ -209,10 +209,17 @@ Deliberately thin — the real behavior lives in the core handlers and REST rout - connects to the C2D engines for compute services. ### 3.3 Database -The metadata DB backend is pluggable — **Typesense or Elasticsearch** (chosen by `DB_TYPE`) -for DDOs, indexer state, logs, orders, ddoState, access lists, and escrow events. -**SQLite** is always used (via Node's built-in `node:sqlite`, no native addon) for the nonce, -config, C2D-job, and auth-token DBs — so the node works even with no metadata DB configured. +The node keeps all its data in one database, selected with `DB_TYPE`: **SQLite** (the +default, files in `databases/`, one node instance), **PostgreSQL** or **Elasticsearch** (a +server, for several node instances; `DB_URL`, `DB_USERNAME`, `DB_PASSWORD`). The database +holds the index data (DDOs, indexer state, logs, orders, ddoState, access lists, escrow +events) and the node state (nonces, node config, auth tokens, buckets, node metrics, compute +jobs, service jobs, image records, service locks). There is no mode that runs without it. + +The code outside `src/components/database` does not know which database is in use: it gets +the 13 stores of `DatabaseStores`, and `DatabaseFactory` is the only place that selects the +backend. SQLite and PostgreSQL share one set of stores (`database/sql/`) through the +`SqlClient` interface. See [database.md](database.md). ### 3.4 Supporting components - **Auth** — JWT-based auth-token issuance/validation (`JWT_SECRET`) as an alternative to diff --git a/docs/Logs.md b/docs/Logs.md index 2fcf0d053..cfaf48c24 100644 --- a/docs/Logs.md +++ b/docs/Logs.md @@ -8,10 +8,10 @@ Refer to Logs section in the [environmental variables documentation](env.md#logs | --- | --- | --- | --- | | Console (stdout) | on | `LOG_LEVEL` (`info` in production, `debug` otherwise) | `LOG_CONSOLE` (`false` turns it off) | | Files on disk (`logs/`) | on when `LOG_DB` is off | `LOG_FILES_LEVEL` (default `info`), 30 days retention | `LOG_FILES`, `LOG_FILES_*` | -| Log database | off | `LOG_LEVEL` | `LOG_DB` (needs `DB_URL`), `LOG_RETENTION_TIME` | +| Log database | off | `LOG_LEVEL` | `LOG_DB`, `LOG_RETENTION_TIME` | - The console stays on whatever `LOG_FILES` and `LOG_DB` are set to. Use `LOG_CONSOLE=false` to turn it off. -- If `LOG_DB=true` but the log database cannot be used (no `DB_URL`, init failed, degraded mode), the node falls back to files on disk and logs a warning. `LOG_FILES=false` disables that fallback. +- If `LOG_DB=true` but the database cannot be initialized, the node writes the failure to files on disk (to the console when `LOG_FILES=false`) and then exits. - In Docker, mount `./logs:/usr/src/app/logs` (the quickstart compose file does) or the files are lost when the container is recreated. See [dockerDeployment.md](dockerDeployment.md). - With PM2, files are written to `logs/` in the node's working directory. PM2 also keeps stdout in `~/.pm2/logs`, which is not size-limited unless `pm2-logrotate` is installed. diff --git a/docs/Ocean Node.postman_collection.json b/docs/Ocean Node.postman_collection.json index a4bbcfdfa..aefb525ae 100644 --- a/docs/Ocean Node.postman_collection.json +++ b/docs/Ocean Node.postman_collection.json @@ -91,7 +91,7 @@ ] }, { - "name": "Aquarius (DDO)", + "name": "DDO and query", "item": [ { "name": "Get DDO by DID", @@ -99,14 +99,12 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/aquarius/assets/ddo/{{did}}", + "raw": "{{baseUrl}}/api/ddo/{{did}}", "host": [ "{{baseUrl}}" ], "path": [ "api", - "aquarius", - "assets", "ddo", "{{did}}" ] @@ -114,27 +112,6 @@ "description": "Retrieve the full DDO for a DID. Append an optional `/true` path segment to force a fresh lookup." } }, - { - "name": "Get DDO Metadata by DID", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/aquarius/assets/metadata/{{did}}", - "host": [ - "{{baseUrl}}" - ], - "path": [ - "api", - "aquarius", - "assets", - "metadata", - "{{did}}" - ] - }, - "description": "Retrieve DDO metadata for a DID. Optional trailing `/true` forces a fresh lookup." - } - }, { "name": "Query Assets", "request": { @@ -147,18 +124,15 @@ ], "body": { "mode": "raw", - "raw": "{\n \"query\": {\n \"match_all\": {}\n },\n \"from\": 0,\n \"size\": 10\n}" + "raw": "{\n \"filter\": { \"chainIds\": [8996] },\n \"page\": 1,\n \"size\": 10\n}" }, "url": { - "raw": "{{baseUrl}}/api/aquarius/assets/metadata/query", + "raw": "{{baseUrl}}/api/query", "host": [ "{{baseUrl}}" ], "path": [ "api", - "aquarius", - "assets", - "metadata", "query" ] }, @@ -171,15 +145,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/aquarius/state/ddo?did={{did}}", + "raw": "{{baseUrl}}/api/ddo/state?did={{did}}", "host": [ "{{baseUrl}}" ], "path": [ "api", - "aquarius", - "state", - "ddo" + "ddo", + "state" ], "query": [ { @@ -221,14 +194,12 @@ "raw": "{\n \"ddo\": {},\n \"publisherAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" }, "url": { - "raw": "{{baseUrl}}/api/aquarius/assets/ddo/validate", + "raw": "{{baseUrl}}/api/ddo/validate", "host": [ "{{baseUrl}}" ], "path": [ "api", - "aquarius", - "assets", "ddo", "validate" ] @@ -2326,7 +2297,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"command\": \"query\",\n \"query\": { \"query\": { \"match_all\": {} }, \"from\": 0, \"size\": 10 }\n}" + "raw": "{\n \"command\": \"query\",\n \"query\": { \"filter\": { \"chainIds\": [8996] }, \"page\": 1, \"size\": 10 }\n}" }, "url": { "raw": "{{baseUrl}}/directCommand", diff --git a/docs/Publishing.md b/docs/Publishing.md index ce8ecd0a0..7b9554ec6 100644 --- a/docs/Publishing.md +++ b/docs/Publishing.md @@ -18,7 +18,7 @@ In the usual setup, every Ocean Node that follows the network is free to index e You submit the metadata transaction (create/update) so the asset is anchored on the target chain. 4. **Indexing** - Every node that indexes that chain processes the relevant events and stores the DDO in its local index (e.g. Typesense/Elasticsearch), so the asset becomes discoverable through those nodes’ APIs. + Every node that indexes that chain processes the relevant events and stores the DDO in its local index (Elasticsearch), so the asset becomes discoverable through those nodes’ APIs. Nothing in this flow *requires* an allowlist: if your nodes do not set the isolation options below, they behave as part of the open network. diff --git a/docs/database.md b/docs/database.md index 04e04714c..670615468 100644 --- a/docs/database.md +++ b/docs/database.md @@ -1,54 +1,222 @@ # Ocean Node Database Configuration -Ocean Node can be run with two types of databases: Elasticsearch or Typesense, or with no database at all (using a NoSQL setup). This flexibility allows you to configure the node based on your infrastructure needs. +Ocean Node keeps all its data in one database. You must select the database with `DB_TYPE`: +a node with no `DB_TYPE` does not start. The node does not get the type from `DB_URL`, and it +does not check the URL: the driver of `DB_TYPE` connects to it as given. + +| `DB_TYPE` | When to use it | What the node needs | +| --------------- | --------------------------------------------- | -------------------------------------------------- | +| `sqlite` | One node instance. No database container. | Nothing. The files are in the `databases/` folder. | +| `postgresql` | Several node instances on one database. | `DB_URL`, `DB_USERNAME`, `DB_PASSWORD` | +| `elasticsearch` | Several node instances, or a large catalogue. | `DB_URL`, `DB_USERNAME`, `DB_PASSWORD` | + +There is no mode that runs without a database. The node tries to open its database a few +times at start (`DB_INIT_MAX_ATTEMPTS`), then stops. + +The database holds two groups of data: + +- **Index data**, which the indexer reads from the chains: DDOs, DDO states, orders, access + lists, escrow events, and the last indexed block of each chain. The node can make this + data again from the chains. The logs are in the database too when `LOG_DB` is on; the + node cannot make them again. +- **Node state**: compute jobs, service jobs, Docker image records, service locks, nonces, + auth tokens, node settings, persistent-storage buckets and node metrics. ## Database Configuration -Depending on the database type you choose, you will need to set specific environment variables. Ocean Node supports either Elasticsearch or Typesense as the database for storing the various node components. +### SQLite -### 1. Set the Environment Variables +Set `DB_TYPE="sqlite"`. The node makes these files in `databases/` (next to `p2p-store`, the libp2p +datastore): - • For Typesense, you need to set the following environment variables: +| File | Content | +| -------------------------- | -------------------------------------------------------- | +| `c2dDatabase.sqlite` | compute jobs, service jobs, image records, service locks | +| `nonceDatabase.sqlite` | nonces | +| `authTokenDatabase.sqlite` | auth tokens | +| `config.sqlite` | node settings | +| `persistentStorage.sqlite` | persistent-storage buckets | +| `nodeMetrics.sqlite` | node metrics history | +| `index.sqlite` | the index data and the logs | -```bash -DB_TYPE=typesense -DB_URL="http://localhost:8108/?apiKey=xyz" # Example URL when using Barge for Typesense -``` +One `databases/` folder is for one node instance. Two instances must not use the same folder. +In Docker, mount a volume on `/usr/src/app/databases`, or the data is lost when the container +is replaced. - • For Elasticsearch, you need to set: +### PostgreSQL ```bash -DB_TYPE=elasticsearch -DB_URL="http://localhost:9200" # Example URL when using Barge for Elasticsearch +DB_TYPE="postgresql" +DB_URL="postgresql://localhost:5432/ocean" # host, port and database name +DB_USERNAME="ocean" +DB_PASSWORD="changeme" ``` -Ensure that the correct DB_TYPE is specified as either typesense or elasticsearch depending on your chosen setup. - -### 2. Starting Ocean Barge - -To run Ocean Node with the appropriate database, you need to start Barge with specific flags. +The node makes its tables in the database of the URL. Several instances can use one database. +For local development, `docker-compose -f postgresql-compose.yml up -d` starts PostgreSQL +with these values. - • To run Ocean Node with Typesense, use the following command: +### Elasticsearch ```bash -./start_ocean.sh --no-aquarius --no-provider --no-dashboard --with-c2d --with-typesense --no-elasticsearch +DB_TYPE="elasticsearch" +DB_URL="http://localhost:9200" +DB_USERNAME="elastic" +DB_PASSWORD="changeme" ``` - • To run Ocean Node with Elasticsearch, use the following command: +The node makes its indices: the DDO indices (`op_ddo_*`), `indexer`, `logs`, `order`, +`ddo_state`, `access_list`, `escrow`, and the node state indices `nonce`, `node_config`, +`auth_tokens`, `buckets`, `node_metrics_samples`, `node_metrics_hourly`, `c2d_jobs`, +`service_jobs`, `docker_images` and `service_locks`. For local development, +`docker-compose -f elasticsearch-compose.yml up -d` starts Elasticsearch with these values. +Barge starts one too: ```bash ./start_ocean.sh --no-aquarius --no-provider --no-dashboard --with-c2d ``` -By specifying these flags, you can configure Ocean Node to work with either Typesense or Elasticsearch databases, depending on your requirements. +## Do not publish the database + +The database holds the node state, and the database user of the node has full access. Do not +publish the port of PostgreSQL or Elasticsearch to the internet: only the node must reach it. +`scripts/ocean-node-quickstart.sh` makes a compose file where the database has no published +port, and a random database password for each new deployment. The development compose files +bind the port to `127.0.0.1` and use the password `changeme`. + +A database server reads its password only when it creates its data. To use your own password, +change it for the node and for the database before the first start. For the same reason, +remove the data volume of a database before you select that database again in the quickstart +script: the script makes a new password, and the old volume keeps the old one. + +Frontends read DDOs through the node: `POST /api/query` (or the `query` command) to search, +and `GET /api/ddo/:did` (or the `getDDO` command) for one DDO. + +## DDO search + +The query of the node is the same on all databases: a JSON object with a type, typed +filters, a sort key and a page. It has the searches that the markets use, and nothing more. +Each database runs it with its own fixed queries: there is no Elasticsearch query syntax, and +an Elasticsearch body (with `query`, `from` or `index`) gives HTTP 400. + +```json +{ + "type": "assets", + "filter": { "chainIds": [8996], "types": ["dataset"], "text": "weather" }, + "sort": { "by": "created", "order": "desc" }, + "page": 1, + "size": 21 +} +``` + +The answer is `{ "results": [...], "totalResults": 42 }`: one list for the DDOs of all +versions (v4 and v5), sorted and paged over all of them. + +| Field | Value | +| -------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `type` | `assets` (the default), `orders` or `tags` | +| `filter` | The filters below. With `orders`: `{ "consumer": "0x…" }` or `{ "payer": "0x…" }` | +| `sort` | `assets` only. `by`: `created` (the event block), `orders`, `price`, `tokenSymbol` or `relevance`; `order`: `asc` or `desc` (the default) | +| `page` | Starts at 1. `page` × `size` is 10000 at most | +| `size` | 0 to 10000, 1000 when absent. With `tags`: the largest number of tags, 1 to 10000 | + +The asset filters. A filter that is not set adds no condition; all filters that are set must +hold. Each filter covers the field of a DDO v4 and of a DDO v5 (`credentialSubject.…`). + +| Filter | Matches | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `chainIds` | `chainId` is one of the values | +| `versions` | the DDO `version` is one of the values, for example `"5.0.0"` | +| `ids`, `excludeIds` | the DID is (is not) one of the values | +| `excludeNftAddresses` | `nftAddress` is not one of the values | +| `owners` | `indexedMetadata.nft.owner` is one of the values | +| `anyOf: [{ ids, owners, datatokenAddresses }]` | in each group, one of the lists matches (DID, owner, a service datatoken). All groups must hold | +| `types` | `metadata.type`: `dataset`, `algorithm` | +| `serviceTypes` | a service has the type: `access`, `compute` | +| `datatokenAddresses` | a service has one of the datatokens | +| `tags` | `metadata.tags` has one of the values | +| `text` | each word is in the name, description, author, tags, owner, or datatoken name, symbol or address; or the whole text is the DID | +| `nftStates` | `indexedMetadata.nft.state` is one of the values | +| `includeUnlisted` | `true` also gives the unlisted assets (state 5). Not set: they are hidden, also when `nftStates` has 5 | +| `includePurgatory` | `true` also gives the assets in purgatory. Not set: they are hidden | +| `priceTypes`, `excludePriceTypes` | `indexedMetadata.stats.prices.type` is (is not) one of the values | +| `createdAfter` | `metadata.created` is at or after the ISO date | +| `serviceEndpoints` | a service has one of the endpoints | +| `dataspace` | `credentialSubject.dataspace` is the value | +| `trustedAlgorithm: { did, serviceId }` | a compute service trusts the algorithm (`"*"`: all algorithms) | +| `trustedAlgorithmPublisher` | a compute service trusts the publisher (`"*"`: all publishers) | +| `indexedOnly` | the DDO has `indexedMetadata` | + +`type: "tags"` answers the distinct tags of the assets that match the filter, the most used +first: `{ "results": [{ "tag": "weather", "count": 2 }], "totalResults": 1 }`. + +An address (owner, datatoken, NFT, consumer, payer) matches in any letter case. The databases +differ in these points: + +- `text`: on SQLite and PostgreSQL a word matches when a value contains it; on Elasticsearch + it matches a word of a value, or the start of one. +- `relevance` sorts by the text score on Elasticsearch, then the newest first; SQLite and + PostgreSQL have no score and give the newest first. +- `price` is stored as text: SQLite and PostgreSQL sort it as a number, Elasticsearch as text. +- The tags of a `tags` answer are in lower case on SQLite and PostgreSQL. Their comparisons + ignore the letter case; on Elasticsearch a tag, a type and an id must match exactly. + +## Upgrade from an older release, and a change of database + +The older releases had no `DB_TYPE`. Add it to the compose file or `.env` of every node, or +the node does not start. + +The older releases kept the node state in the SQLite files of `databases/` also when the +index data was in Elasticsearch. The node state does not move from these files to a database +server, and it does not move between databases. You have two choices: + +- Keep the state: set `DB_TYPE=sqlite` (and remove `DB_URL`). The node makes the index data + again in SQLite. +- Start with empty state on the database server: stop the node when it has no running jobs + or services, then set `DB_TYPE` to `postgresql` or `elasticsearch`. The node does not use + the SQLite files that stay in `databases/`. With no nonces, a signed request that the node + accepted before can be accepted again, and the paid jobs in the SQLite files are not + claimed. + +A node that used Elasticsearch in an older release keeps its Elasticsearch, but three indices +changed: `state` is now `ddo_state`, `access_list` keeps `chainId` as a `long`, and +`op_ddo_v5.0.0` indexes the trusted algorithms of a compute service (the DDO search filters +on them). The node does not change an index that exists. Before the first start, delete the +indices `state`, `access_list` and `op_ddo_v5.0.0`. The `logs` index stays as it is, with +its logs. The node state +of the older release is in SQLite, so the node settings on Elasticsearch hold no index +version: at the first start the node indexes all chains again, and makes these indices +again. + +The DDO search of the older releases took an Elasticsearch query body. It now takes the +query of the "DDO search" section on all databases, and answers `{ results, totalResults }`: +a client that sends an Elasticsearch body gets HTTP 400. + +A compose file of an older release can have a `DB_URL` for Typesense, which the node no longer +supports. Remove the `DB_URL` line and the Typesense service, and set `DB_TYPE=sqlite`: the +node keeps its state in `databases/` and makes the index data again from the chains. + +## Auth tokens + +The node stores only the sha256 hash of each auth token, never the token itself. A daily cron job deletes expired and +invalidated tokens. + +## Node metrics history + +Node metrics history is kept per instance. Instances that share one `PRIVATE_KEY` share one peer id, so the node +keys its metrics by `INSTANCE_ID`, or by the hostname when `INSTANCE_ID` is not set. A history +request answers from the instance that receives it. In containers the hostname changes on +redeploy, so container deployments set `INSTANCE_ID` (see [env.md](env.md)). Retention +runs per instance id, so the node never sweeps the rows of a retired `INSTANCE_ID`: set +`INSTANCE_ID` to a stable value. ## Runtime metrics on C2D job records While a compute job or Service-on-Demand container runs, the node periodically samples live Docker (and, on NVIDIA hosts, NVML) metrics — CPU, RAM, disk usage vs quota, network, block I/O, PID count, CPU throttling, memory peak, exit/OOM info, and per-GPU utilization/memory — -and stores the latest snapshot on the job record (inside the existing JSON `body` blob of the -SQLite `c2djobs` / `service_jobs` tables). A **final** snapshot is written at termination +and stores the latest snapshot on the job record (inside the JSON `body` of the compute job +or service job). A **final** snapshot is written at termination (publishing results, quota kill, service stop/restart, or unexpected container death) so peak/exit metrics remain queryable after the container is gone. No schema migration is needed; pre-upgrade records simply lack the field. diff --git a/docs/dockerDeployment.md b/docs/dockerDeployment.md index 133d2d45c..d3e2ef97e 100644 --- a/docs/dockerDeployment.md +++ b/docs/dockerDeployment.md @@ -59,7 +59,6 @@ e) confirm that docker containers are running $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 188bf1eec4c1 oceanprotocol/ocean-node:latest "npm run start" 11 seconds ago Up 10 seconds 0.0.0.0:8000->8000/tcp, :::8000->8000/tcp, 0.0.0.0:9000-9003->9000-9003/tcp, :::9000-9003->9000-9003/tcp ocean-node -858a59502302 typesense/typesense:26.0 "/opt/typesense-serv…" 17 seconds ago Up 10 seconds 0.0.0.0:8108->8108/tcp, :::8108->8108/tcp typesense ``` ## Upgrade Ocean Node @@ -95,10 +94,11 @@ $ docker run -d \ oceanprotocol/ocean-node:latest ``` -This is not just a convenience. `/usr/src/app/databases` holds: - -- the SQLite databases - nonce tracking, node config, C2D job records and auth tokens; -- `databases/p2p-store`, the libp2p LevelDB datastore. +This is not just a convenience. `/usr/src/app/databases` holds `databases/p2p-store`, the +libp2p LevelDB datastore. With `DB_TYPE=sqlite` it also holds all the data of +the node: the index data and the node state (nonces, node config, C2D and service jobs, auth +tokens, buckets, metrics). With PostgreSQL or Elasticsearch that data is in the database +server, so keep the data volume of the server persistent too. See [database.md](database.md). The datastore is what makes the node keep answering for the assets it holds. Its own DHT provider records are written from three places and no others: when the indexer sees a new @@ -128,10 +128,10 @@ Ownership is handled for you: the image creates the directory owned by the unpri user it runs as, and `docker-entrypoint.sh` re-applies ownership at startup, which is what makes a bind mount of a host directory work regardless of who created it on the host. -The two development compose files in the repository root, `typesense-compose.yml` and -`elasticsearch-compose.yml`, define only the database service (Typesense or Elasticsearch). -They do not contain an `ocean-node` service, which is what the "run the node from npm" -workflow in the README expects. To run the node in Docker, use the compose file generated by +The development compose files in the repository root, `postgresql-compose.yml` and +`elasticsearch-compose.yml`, define only a database service, for a node that uses that +database. They do not contain an `ocean-node` service, which is what the "run the node from +npm" workflow in the README expects. To run the node in Docker, use the compose file generated by `scripts/ocean-node-quickstart.sh` (see below). ### Log files (recommended) @@ -180,7 +180,7 @@ the logs back see [Logs.md](Logs.md). Additional notes: -- the docker compose file generated will have the following format. For all available configurations, refer to the [Environment Variables](https://github.com/oceanprotocol/ocean-node/blob/main/docs/env.md) documentation +- the script asks for the database (`DB_TYPE` is required). With SQLite, the docker compose file generated will have the following format. For all available configurations, refer to the [Environment Variables](https://github.com/oceanprotocol/ocean-node/blob/main/docs/env.md) documentation ```yaml services: @@ -202,9 +202,11 @@ services: - '9002:9002' - '9003:9003' environment: + DB_TYPE: 'sqlite' PRIVATE_KEY: '<>' RPCS: '{"1":{"rpc":"https://ethereum-rpc.publicnode.com","fallbackRPCs":["https://rpc.ankr.com/eth","https://1rpc.io/eth","https://eth.api.onfinality.io/public"],"chainId":1,"network":"mainnet","chunkSize":100},"10":{"rpc":"https://mainnet.optimism.io","fallbackRPCs":["https://optimism-mainnet.public.blastapi.io","https://rpc.ankr.com/optimism","https://optimism-rpc.publicnode.com"],"chainId":10,"network":"optimism","chunkSize":100},"137":{"rpc":"https://polygon-rpc.com/","fallbackRPCs":["https://polygon-mainnet.public.blastapi.io","https://1rpc.io/matic","https://rpc.ankr.com/polygon"],"chainId":137,"network":"polygon","chunkSize":100},"23294":{"rpc":"https://sapphire.oasis.io","fallbackRPCs":["https://1rpc.io/oasis/sapphire"],"chainId":23294,"network":"sapphire","chunkSize":100},"23295":{"rpc":"https://testnet.sapphire.oasis.io","chainId":23295,"network":"sapphire-testnet","chunkSize":100},"11155111":{"rpc":"https://eth-sepolia.public.blastapi.io","fallbackRPCs":["https://1rpc.io/sepolia","https://eth-sepolia.g.alchemy.com/v2/demo"],"chainId":11155111,"network":"sepolia","chunkSize":100},"11155420":{"rpc":"https://sepolia.optimism.io","fallbackRPCs":["https://endpoints.omniatech.io/v1/op/sepolia/public","https://optimism-sepolia.blockpi.network/v1/rpc/public"],"chainId":11155420,"network":"optimism-sepolia","chunkSize":100}}' - DB_URL: 'http://typesense:8108/?apiKey=xyz' + # the node metrics history is kept for this id + INSTANCE_ID: 'ocean-node' IPFS_GATEWAY: 'https://ipfs.io/' ARWEAVE_GATEWAY: 'https://arweave.net/' # LOAD_INITIAL_DDOS: '' @@ -261,27 +263,66 @@ services: options: max-size: '50m' max-file: '5' + +volumes: + node-sqlite: + driver: local + +networks: + ocean_network: + driver: bridge +``` + +- with PostgreSQL or Elasticsearch, the script adds the database settings to the node, a + database service and its volume. The database service has no `ports` entry: only the node + can reach it. The password is random, made by the script, and is the same for the node and + for the database. For PostgreSQL: + +```yaml +services: + ocean-node: + environment: + DB_TYPE: 'postgresql' + DB_URL: 'postgresql://postgresql:5432/ocean' + DB_USERNAME: 'ocean' + DB_PASSWORD: '<>' depends_on: - - typesense + - postgresql - typesense: - image: typesense/typesense:26.0 - container_name: typesense - ports: - - '8108:8108' + postgresql: + image: postgres:17 + container_name: postgresql + restart: on-failure networks: - ocean_network volumes: - - typesense-data:/data - command: '--data-dir /data --api-key=xyz' + - postgresql-data:/var/lib/postgresql/data + environment: + POSTGRES_DB: 'ocean' + POSTGRES_USER: 'ocean' + POSTGRES_PASSWORD: '<>' volumes: - typesense-data: - driver: local - node-sqlite: + postgresql-data: driver: local +``` -networks: - ocean_network: - driver: bridge +- for Elasticsearch the node gets `DB_TYPE: 'elasticsearch'`, `DB_URL: +'http://elasticsearch:9200'` and `DB_USERNAME: 'elastic'`, and the service is: + +```yaml +elasticsearch: + image: elasticsearch:8.5.1 + container_name: elasticsearch + restart: on-failure + networks: + - ocean_network + volumes: + - elasticsearch-data:/usr/share/elasticsearch/data + environment: + ES_JAVA_OPTS: '-Xms512m -Xmx512m' + discovery.type: 'single-node' + ELASTIC_PASSWORD: '<>' + xpack.security.enabled: 'true' + xpack.security.http.ssl.enabled: 'false' ``` diff --git a/docs/env.md b/docs/env.md index dcbed34d4..c1b3ebbc6 100644 --- a/docs/env.md +++ b/docs/env.md @@ -7,7 +7,6 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `PRIVATE_KEY` (Required): The private key for the node, required for node operations. Example: `"0x1d751ded5a32226054cd2e71261039b65afb9ee1c746d055dd699b1150a5befc"` - `CONFIG_PATH`: Absolute path to JSON config file - `RPCS`: JSON object defining RPC endpoints for various networks. Example: `"{ \"11155420\":{ \"rpc\":\"https://sepolia.optimism.io\", \"fallbackRPCs\": [\"https://public.stackup.sh/api/v1/node/optimism-sepolia\"], \"chainId\": 11155420, \"network\": \"optimism-sepolia\", \"chunkSize\": 1000 }}"` -- `DB_URL`: URL for connecting to the database. Required for running a database with the node. Example: `"http://localhost:8108/?apiKey=xyz"` - `IPFS_GATEWAY`: The gateway URL for IPFS, used for downloading files from IPFS. Example: `"https://ipfs.io/"` - `ARWEAVE_GATEWAY`: The gateway URL for Arweave, used for downloading files from Arweave. Example: `"https://arweave.net/"` - `LOAD_INITIAL_DDOS`: If set, the node will load initial DDOs from JSON files at startup. This is useful for testing or bootstrapping the network with predefined data. Example: `false` @@ -25,7 +24,7 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `INDEXER_NETWORKS`: Specifies the networks the Indexer will crawl. If not set, the Indexer will index all networks defined in the RPCS environment variable. If set to an empty string, indexing will be disabled. Example: `[1, 137]` - `ALLOWED_ADMINS`: Sets the public address of accounts which have access to admin endpoints e.g. shutting down the node. Example: `"[\"0x967da4048cD07aB37855c090aAF366e4ce1b9F48\",\"0x388C818CA8B9251b393131C08a736A67ccB19297\"]"` - `ALLOWED_ADMINS_LIST`: Array of access list addresses (per chain) for accounts that have access to admin endpoints. Example: `"{ \"8996\": [\"0x123\",\"0x456\"]"` -- `MAINTENANCE_MODE`: Boots the node directly into maintenance (drain) mode. The value is read at startup and ONLY the exact strings `true` and `false` are honored (case-insensitive); any other non-empty value (e.g. `1`, `yes`, a typo) is IGNORED with a warning and the node falls back to the persisted flag. The env value WINS at boot and is written through to the SQLite config DB, so **`MAINTENANCE_MODE=false` overrides a persisted `true`** — a leftover `false` will clear a drain you enabled at runtime. When set to `true` the node starts already rejecting new compute/service requests while still serving already-committed work (see the `setMaintenance` admin command and `status.maintenance` in [API.md](API.md)). **Sharp edge:** the flag is persisted and STICKY across restarts, so returning to normal service requires BOTH an explicit `setMaintenance { enabled: false }` admin command AND unsetting `MAINTENANCE_MODE` before the next restart (otherwise the node boots back into maintenance). There is no TTL and no auto-exit. Default is off (if unset, the persisted flag is used). Example: `true` +- `MAINTENANCE_MODE`: Boots the node directly into maintenance (drain) mode. The value is read at startup and ONLY the exact strings `true` and `false` are honored (case-insensitive); any other non-empty value (e.g. `1`, `yes`, a typo) is IGNORED with a warning and the node falls back to the persisted flag. The env value WINS at boot and is written through to the node settings in the database, so **`MAINTENANCE_MODE=false` overrides a persisted `true`** — a leftover `false` will clear a drain you enabled at runtime. When set to `true` the node starts already rejecting new compute/service requests while still serving already-committed work (see the `setMaintenance` admin command and `status.maintenance` in [API.md](API.md)). **Sharp edge:** the flag is persisted and STICKY across restarts, so returning to normal service requires BOTH an explicit `setMaintenance { enabled: false }` admin command AND unsetting `MAINTENANCE_MODE` before the next restart (otherwise the node boots back into maintenance). There is no TTL and no auto-exit. Default is off (if unset, the persisted flag is used). Example: `true` - `RATE_DENY_LIST`: Blocked list of IPs and peer IDs. Example: `"{ \"peers\": [\"16Uiu2HAkuYfgjXoGcSSLSpRPD6XtUgV71t5RqmTmcqdbmrWY9MJo\"], \"ips\": [\"127.0.0.1\"] }"` - `MAX_REQ_PER_MINUTE`: Per-caller request ceiling, enforced over a fixed 60-second window and keyed by the caller the transport provides — client IP for HTTP, peer id for P2P (never a value taken from the request body). Applies to both front doors and at both the middleware and handler level. Default: `30`. Example: `30` - `MAX_CONNECTIONS_PER_MINUTE`: Node-wide ceiling on the total number of accepted requests processed per minute, counted through a single shared global counter. It spans **both** HTTP API requests **and** accepted P2P protocol commands, not HTTP alone. Default: `120`. Size it for your combined HTTP + P2P traffic and keep it comfortably above `MAX_REQ_PER_MINUTE` × expected concurrent clients; setting it equal to `MAX_REQ_PER_MINUTE`, or forgetting P2P load, lets normal traffic trip the global cap and briefly affect everyone. @@ -47,17 +46,18 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ ## Database -- `DB_URL`: URL for connecting to the database. Required for running a database with the node. Example: `"http://localhost:8108/?apiKey=xyz"` -- `DB_USERNAME`: Username for database authentication. Optional if not using authentication. Example: `"elastic"` -- `DB_PASSWORD`: Password for database authentication. Optional if not using authentication. Example: `"password123"` -- `ELASTICSEARCH_REQUEST_TIMEOUT`: Request timeout in milliseconds for Elasticsearch operations. Default is `60000`. Example: `60000` -- `ELASTICSEARCH_PING_TIMEOUT`: Ping timeout in milliseconds for Elasticsearch health checks. Default is `5000`. Example: `5000` +- `DB_TYPE`: **Required.** The database of the node: `sqlite`, `postgresql` or `elasticsearch`. With no `DB_TYPE` the node does not start; it does not get the type from `DB_URL`. SQLite needs no other `DB_*` variable. An empty `DB_*` variable counts as not set. See [database.md](database.md). Example: `"sqlite"` +- `DB_URL`: URL of the database server. Required when `DB_TYPE` is `postgresql` or `elasticsearch`; not used for `sqlite`. The node does not check the URL: the driver of `DB_TYPE` connects to it as given. Examples: `"postgresql://localhost:5432/ocean"`, `"http://localhost:9200"` +- `DB_USERNAME`: Username for database authentication. Required when `DB_TYPE` is `postgresql` or `elasticsearch`. Example: `"elastic"` +- `DB_PASSWORD`: Password for database authentication. Required when `DB_TYPE` is `postgresql` or `elasticsearch`. Example: `"password123"` +- `ELASTICSEARCH_REQUEST_TIMEOUT`: Request timeout in milliseconds for Elasticsearch operations. Default is `30000`. Example: `30000` +- `ELASTICSEARCH_PING_TIMEOUT`: Ping timeout in milliseconds for Elasticsearch health checks. Default is `3000`. Example: `3000` - `ELASTICSEARCH_RESURRECT_STRATEGY`: Strategy for bringing failed Elasticsearch nodes back online. Options are 'ping', 'optimistic', or 'none'. Default is `ping`. Example: `"ping"` -- `ELASTICSEARCH_MAX_RETRIES`: Maximum number of retry attempts for failed Elasticsearch operations. Default is `5`. Example: `5` +- `ELASTICSEARCH_MAX_RETRIES`: Maximum number of retry attempts for failed Elasticsearch operations. Default is `3`. Example: `3` - `ELASTICSEARCH_SNIFF_ON_START`: Enable cluster node discovery on Elasticsearch client startup. Default is `true`. Example: `true` - `ELASTICSEARCH_SNIFF_INTERVAL`: Interval in milliseconds for periodic cluster health monitoring and node discovery. Set to 'false' to disable. Default is `30000`. Example: `30000` - `ELASTICSEARCH_SNIFF_ON_CONNECTION_FAULT`: Enable automatic cluster node discovery when connection faults occur. Default is `true`. Example: `true` -- `ELASTICSEARCH_HEALTH_CHECK_INTERVAL`: Interval in milliseconds for proactive connection health monitoring. Default is `60000`. Example: `60000` +- `ELASTICSEARCH_HEALTH_CHECK_INTERVAL`: Interval in milliseconds for proactive connection health monitoring. Default is `15000`. Example: `15000` - `DB_INIT_MAX_ATTEMPTS`: Maximum number of database initialization attempts at node startup. Raise it when the database container/pod starts slower than the node. Set to `1` to disable retrying. All three `DB_INIT_*` variables are validated as integers `>= 1`; a non-numeric or out-of-range value fails configuration validation and the node refuses to start. Default is `10`. Example: `10` - `DB_INIT_RETRY_DELAY`: Initial delay in milliseconds before retrying a failed database initialization. Doubles after each attempt. Default is `2000`. Example: `2000` - `DB_INIT_MAX_RETRY_DELAY`: Upper bound in milliseconds for the database initialization retry backoff. Default is `30000`. Example: `30000` @@ -82,7 +82,7 @@ Console, disk and database are independent log sinks. See [Logs.md](Logs.md) for | `LOG_FILES_RETENTION_DAYS` | `30` | Days of log files to keep. Older files (including `.gz` archives) are deleted when the log rotates. Integer, minimum `1` | | `LOG_FILES_MAX_SIZE` | `20m` | Size at which one module's daily file splits into `.1`, `.2`, ... parts. Accepts `k`, `m` or `g` | | `LOG_FILES_DIR` | `logs` | Directory for log files, relative to the working directory (`/usr/src/app/logs` in Docker) | -| `LOG_DB` | `false` | Write logs to the noSQL database (needs `DB_URL`). Turning it on switches off default file logging unless `LOG_FILES=true` | +| `LOG_DB` | `false` | Write logs to the database of the node. Turning it on switches off default file logging unless `LOG_FILES=true` | | `LOG_RETENTION_TIME` | `2592000000` | Retention of logs in the database, in milliseconds (30 days). DB logs only | Invalid levels, retention values or sizes fall back to their defaults and log a warning at startup. Files are written to `/-YYYY-MM-DD.log` (UTC days); see [Logs.md](Logs.md#log-files-on-disk). @@ -208,11 +208,12 @@ setting it through the environment reaches both. ## Node Metrics History -Powers the `getNodeMetrics` (live snapshot) and `getNodeMetricsHistory` (hourly averages) commands / REST routes. The history layer is SQLite-backed (`databases/nodeMetrics.sqlite`) so it works even with no metadata DB configured. A minute sampler writes the same per-node aggregate the live command returns into a short-lived raw buffer, an hourly roll-up at minute `:05` averages each complete hour into `node_metrics_hourly`, and a daily sweep drops rows older than the retention window. The sampler **warns and skips** (persists no row) when there is no fresh compute aggregate — i.e. `C2D_METRICS_INTERVAL_SECONDS=0` or no engine has sampled yet — so all-zero rows never skew the averages. The live `getNodeMetrics` command still returns a (zeroed) snapshot in that case. +Powers the `getNodeMetrics` (live snapshot) and `getNodeMetricsHistory` (hourly averages) commands / REST routes. The history layer is stored in the database of the node, per instance (see `INSTANCE_ID` below). A minute sampler writes the same per-node aggregate the live command returns into a short-lived raw buffer, an hourly roll-up at minute `:05` averages each complete hour into `node_metrics_hourly`, and a daily sweep drops rows older than the retention window. The sampler **warns and skips** (persists no row) when there is no fresh compute aggregate — i.e. `C2D_METRICS_INTERVAL_SECONDS=0` or no engine has sampled yet — so all-zero rows never skew the averages. The live `getNodeMetrics` command still returns a (zeroed) snapshot in that case. - `NODE_METRICS_HISTORY_ENABLED`: Enable/disable the node-metrics history sampler + roll-up + retention cron jobs. Defaults to enabled whenever a database is available. Set to `false` (also accepts `0`/`no`) to turn the history layer off; the live `getNodeMetrics` command is unaffected. Example: `true` - `NODE_METRICS_SAMPLE_CRON`: Cron expression for the minute sampler. Defaults to `* * * * *` (every minute). Example: `* * * * *` - `NODE_METRICS_RETENTION_DAYS`: How many days of hourly rows to keep before the daily retention sweep deletes them. Also clamps the range `getNodeMetricsHistory` will return. Defaults to `180` (~6 months). Example: `180` +- `INSTANCE_ID`: Identifies this node instance in the metrics history. Instances that share one `PRIVATE_KEY` share one peer id, so each instance keeps its own metrics under this id. Defaults to the hostname (`os.hostname()`). In containers the hostname changes on every redeploy, which splits the history into fragments, so container deployments must set `INSTANCE_ID` to a fixed value. Example: `node-1` ## Compute diff --git a/docs/persistentStorage.md b/docs/persistentStorage.md index abf081919..47836feb8 100644 --- a/docs/persistentStorage.md +++ b/docs/persistentStorage.md @@ -12,7 +12,7 @@ Key primitives: - **Bucket**: a logical container for files. - **File**: binary content stored inside a bucket. -- **Bucket registry**: a local SQLite table that stores bucket metadata (owner, access lists, createdAt). +- **Bucket registry**: the `buckets` store of the node database (a table on SQLite and PostgreSQL, an index on Elasticsearch) that keeps bucket metadata (owner, access lists, createdAt). --- @@ -25,7 +25,7 @@ Key primitives: - Validates auth (token or signature) and applies high-level authorization checks. - **Persistent storage backends (storage layer)**: `src/components/persistentStorage/*` - - `PersistentStorageFactory`: shared functionality (SQLite bucket registry, access list checks). + - `PersistentStorageFactory`: shared functionality (bucket registry, access list checks). - `PersistentStorageLocalFS`: local filesystem backend. - `PersistentStorageS3`: stub for future S3-compatible backend. @@ -36,11 +36,10 @@ Key primitives: Persistent Storage uses two stores: -1. **Bucket registry (SQLite)** +1. **Bucket registry (node database)** -- File: `databases/persistentStorage.sqlite` -- Table: `persistent_storage_buckets` -- Columns: +- Table or index: `buckets` (id = `bucketId`) +- Fields: - `bucketId` (primary key) - `owner` (address, stored as a string) - `accessListJson` (JSON-encoded access list array) @@ -114,7 +113,7 @@ Access checks happen at two levels: ### Supported today - **Create bucket** - - Creates a bucket id (UUID), persists it in SQLite with `owner` and `accessListJson`, and creates a local directory (localfs). + - Creates a bucket id (UUID), persists it in the `buckets` index with `owner` and `accessListJson`, and creates a local directory (localfs). - **List buckets (by owner)** - Returns buckets from the registry filtered by `owner` (mandatory arg). @@ -255,6 +254,6 @@ Because results are regular bucket files, they can feed the next compute job wit ## Limitations and notes -- The bucket registry is local to the node (SQLite file). If you run multiple nodes, each node’s registry is independent unless you externalize/replicate it. +- The bucket registry lives in the node database. Nodes that share one PostgreSQL or Elasticsearch share the registry; nodes with separate databases have independent registries. - `listBuckets(owner)` requires `owner` and only returns buckets that were created with that owner recorded. - Filenames in `localfs` are constrained (no path separators) to avoid path traversal. diff --git a/docs/services.md b/docs/services.md index 12faf1ae4..1c0abeccf 100644 --- a/docs/services.md +++ b/docs/services.md @@ -183,9 +183,9 @@ restart that had just created it, failing the restart with flight, the expiry sweep simply retries on a later tick. Exclusivity holds **across node processes** too, not just within one: each operation also -takes a lease row in the SQLite `service_locks` table, so two processes sharing the same -`databases/` directory and Docker daemon (e.g. an old container still running during a -redeploy) cannot run conflicting operations on the same service. Leases are heartbeated +takes a lease in the `service_locks` store of the node database, so two node processes +sharing the same database (e.g. an old container still running during a redeploy, or +instances on other hosts) cannot run conflicting operations on the same service. Leases are heartbeated every 30 s while the operation runs; a lease not refreshed for 2 minutes belongs to a crashed process and is stolen automatically, so no manual cleanup is ever needed. diff --git a/docs/testing.md b/docs/testing.md index fb733ac71..6a5ab311f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2,10 +2,60 @@ ## Unit tests +The unit tests run on the database that `DB_TYPE` selects. With no `DB_TYPE` and no `DB_URL` +they run on SQLite and need no database container: + +```bash +export PRIVATE_KEY=0x +npm run test:unit +``` + +The tests load `src/test/.env.test` for the variables that the shell does not set. If that +file has `DB_URL`, `DB_USERNAME` and `DB_PASSWORD` lines for Elasticsearch, the tests run on +Elasticsearch: remove the lines, or set the variables to empty values in the shell +(`DB_TYPE= DB_URL= DB_USERNAME= DB_PASSWORD= npm run test:unit`). + +The SQLite files of the tests are in `databases/` in the repository folder. Do not run the +tests in the folder of a node that you use. + +### On PostgreSQL + +```bash +docker-compose -f postgresql-compose.yml up -d +export DB_TYPE=postgresql DB_URL=postgresql://localhost:5432/ocean +export DB_USERNAME=ocean DB_PASSWORD=changeme +npm run test:unit +``` + +The suites of the SQL stores make a schema of their own for each test and remove it. + +### On Elasticsearch + ```bash +docker-compose -f elasticsearch-compose.yml up -d +export DB_TYPE=elasticsearch DB_URL=http://localhost:9200 +export DB_USERNAME=elastic DB_PASSWORD=changeme npm run test:unit ``` +Some suites use the Elasticsearch client directly and run only with `DB_TYPE=elasticsearch` +(`describeOnElasticsearch` in `src/test/utils/utils.ts`). They delete all documents in the +`auth_tokens`, `c2d_jobs`, `service_jobs`, `service_locks`, `docker_images`, `buckets`, +`node_metrics_samples` and `node_metrics_hourly` indices in `beforeEach`, and write nonces +and the maintenance flag to the `nonce` and `node_config` indices. Never point the tests at +an Elasticsearch that a running node uses. + +On macOS, Elasticsearch sniffing returns the container-internal address, which the host +cannot reach. Turn sniffing off before you run the tests: + +```bash +export ELASTICSEARCH_SNIFF_ON_START=false +export ELASTICSEARCH_SNIFF_INTERVAL=false +export ELASTICSEARCH_SNIFF_ON_CONNECTION_FAULT=false +``` + +CI runs the unit and the integration tests on the three databases. + ## Integration tests To run the integration tests, you should start barge locally. In a separate terminal, clone and start the necessary services using Barge: diff --git a/elasticsearch-compose.yml b/elasticsearch-compose.yml index 98213fc87..7c7944927 100644 --- a/elasticsearch-compose.yml +++ b/elasticsearch-compose.yml @@ -2,16 +2,17 @@ services: elasticsearch: image: elasticsearch:8.5.1 ports: - - 9200:9200 - - 9300:9300 + # loopback only: the database holds the node state + - 127.0.0.1:9200:9200 volumes: - esdata:/usr/share/elasticsearch/data environment: ES_JAVA_OPTS: '-Xms512m -Xmx512m' MAX_MAP_COUNT: '64000' discovery.type: 'single-node' + # for local development only ELASTIC_PASSWORD: 'changeme' - xpack.security.enabled: 'false' + xpack.security.enabled: 'true' xpack.security.http.ssl.enabled: 'false' volumes: esdata: diff --git a/package-lock.json b/package-lock.json index 77be5a31b..16ecfa575 100644 --- a/package-lock.json +++ b/package-lock.json @@ -64,6 +64,7 @@ "multiformats": "^14.0.5", "node-cron": "^4.6.0", "p-limit": "^7.3.3", + "pg": "^8.23.1", "tar": "^7.5.22", "tar-stream": "^3.2.0", "uint8arrays": "^6.1.1", @@ -85,6 +86,7 @@ "@types/lodash": "^4.17.25", "@types/mocha": "^10.0.10", "@types/node": "^26.2.0", + "@types/pg": "^8.23.1", "@types/sinon": "^22.0.0", "@types/tar-stream": "^3.1.4", "@types/yauzl": "^3.4.0", @@ -4961,6 +4963,18 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/qs": { "version": "6.15.1", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz", @@ -11088,6 +11102,95 @@ "dev": true, "license": "MIT" }, + "node_modules/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.1", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.16.1", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz", + "integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz", + "integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz", + "integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -11189,6 +11292,45 @@ "pathe": "^2.0.3" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/powershell-utils": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/powershell-utils/-/powershell-utils-0.1.0.tgz", @@ -12391,6 +12533,15 @@ "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", "license": "ISC" }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -13464,6 +13615,15 @@ "node": ">=4.0" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 0ac82f377..fab5a3aca 100644 --- a/package.json +++ b/package.json @@ -106,6 +106,7 @@ "multiformats": "^14.0.5", "node-cron": "^4.6.0", "p-limit": "^7.3.3", + "pg": "^8.23.1", "tar": "^7.5.22", "tar-stream": "^3.2.0", "uint8arrays": "^6.1.1", @@ -130,6 +131,7 @@ "@types/lodash": "^4.17.25", "@types/mocha": "^10.0.10", "@types/node": "^26.2.0", + "@types/pg": "^8.23.1", "@types/sinon": "^22.0.0", "@types/tar-stream": "^3.1.4", "@types/yauzl": "^3.4.0", diff --git a/postgresql-compose.yml b/postgresql-compose.yml new file mode 100644 index 000000000..d61db0d70 --- /dev/null +++ b/postgresql-compose.yml @@ -0,0 +1,15 @@ +services: + postgresql: + image: postgres:17 + ports: + # loopback only: the database holds the node state + - 127.0.0.1:5432:5432 + volumes: + - pgdata:/var/lib/postgresql/data + environment: + POSTGRES_USER: 'ocean' + # for local development only + POSTGRES_PASSWORD: 'changeme' + POSTGRES_DB: 'ocean' +volumes: + pgdata: diff --git a/schemas/op_ddo_short.json b/schemas/op_ddo_short.json deleted file mode 100644 index a02f6d162..000000000 --- a/schemas/op_ddo_short.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "op_ddo_short", - "enable_nested_fields": true, - "fields": [ - { - "name": ".*", - "type": "auto", - "optional": true - } - ] -} diff --git a/schemas/op_ddo_v1.json b/schemas/op_ddo_v1.json deleted file mode 100644 index 11793cdae..000000000 --- a/schemas/op_ddo_v1.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "op_ddo_v4.1.0", - "enable_nested_fields": true, - "fields": [ - { "name": ".*", "type": "auto", "optional": true } - ] -} diff --git a/schemas/op_ddo_v3.json b/schemas/op_ddo_v3.json deleted file mode 100644 index fea679698..000000000 --- a/schemas/op_ddo_v3.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "op_ddo_v4.3.0", - "enable_nested_fields": true, - "fields": [ - { "name": ".*", "type": "auto", "optional": true } - ] -} diff --git a/schemas/op_ddo_v5.json b/schemas/op_ddo_v5.json deleted file mode 100644 index 89b5b6bb8..000000000 --- a/schemas/op_ddo_v5.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "op_ddo_v4.5.0", - "enable_nested_fields": true, - "fields": [ - { "name": ".*", "type": "auto", "optional": true } - ] -} diff --git a/schemas/op_ddo_v7.json b/schemas/op_ddo_v7.json deleted file mode 100644 index 03a082df7..000000000 --- a/schemas/op_ddo_v7.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "op_ddo_v4.7.0", - "enable_nested_fields": true, - "fields": [{ "name": ".*", "type": "auto", "optional": true }] -} diff --git a/scripts/ocean-node-quickstart.sh b/scripts/ocean-node-quickstart.sh index 1d1515c8e..198368fc7 100755 --- a/scripts/ocean-node-quickstart.sh +++ b/scripts/ocean-node-quickstart.sh @@ -233,6 +233,21 @@ if [ "$enable_tls" == "y" ]; then fi fi +echo "Database of the node:" +echo " 1) SQLite - no database container; one node instance (default)" +echo " 2) PostgreSQL - for several node instances on one database" +echo " 3) Elasticsearch - for several node instances and the full DDO query syntax" +read -p "Select the database [ 1/2/3 ]: " db_choice +case "$db_choice" in + 1|'') DB_TYPE='sqlite' ;; + 2) DB_TYPE='postgresql' ;; + 3) DB_TYPE='elasticsearch' ;; + *) + echo "Invalid database, it must be 1, 2 or 3. Exiting!" + exit 1 + ;; +esac + # Set default compute environments if not already defined if [ -z "$DOCKER_COMPUTE_ENVIRONMENTS" ]; then echo "Setting default DOCKER_COMPUTE_ENVIRONMENTS configuration" @@ -678,6 +693,74 @@ fi echo $DOCKER_COMPUTE_ENVIRONMENTS +# A database server reads its password only when it creates its data. A second run in this +# folder must keep the password of the data volume, so it takes the one in docker-compose.yml. +DB_PASSWORD='' +PREVIOUS_DB_TYPE='' +if [ -f docker-compose.yml ]; then + DB_PASSWORD=$(sed -n "s/^ *DB_PASSWORD: '\(.*\)'.*$/\1/p" docker-compose.yml | head -n 1) + PREVIOUS_DB_TYPE=$(sed -n "s/^ *DB_TYPE: '\(.*\)'.*$/\1/p" docker-compose.yml | head -n 1) + # No DB_TYPE: SQLite, or a release before DB_TYPE (Typesense). That release also kept the + # node state in SQLite files in node-sqlite; Typesense held only the DDO index. + PREVIOUS_DB_TYPE=${PREVIOUS_DB_TYPE:-sqlite} +fi +if [ -z "$DB_PASSWORD" ]; then + DB_PASSWORD=$(head -c 24 /dev/urandom | xxd -p | tr -d '\n') +fi + +# The parts of docker-compose.yml that depend on the database. SQLite needs only DB_TYPE: +# the node keeps its files in the node-sqlite volume. A database server gets no "ports" +# entry, so only the node can reach it. +DB_ENVIRONMENT=" DB_TYPE: 'sqlite'" +DB_DEPENDS_ON='' +DB_SERVICE='' +DB_VOLUME='' +if [ "$DB_TYPE" == "postgresql" ]; then + DB_ENVIRONMENT=" DB_TYPE: 'postgresql' + DB_URL: 'postgresql://postgresql:5432/ocean' + DB_USERNAME: 'ocean' + DB_PASSWORD: '$DB_PASSWORD'" + DB_DEPENDS_ON=" depends_on: + - postgresql" + DB_SERVICE=" postgresql: + image: postgres:17 + container_name: postgresql + restart: on-failure + networks: + - ocean_network + volumes: + - postgresql-data:/var/lib/postgresql/data + environment: + POSTGRES_DB: 'ocean' + POSTGRES_USER: 'ocean' + POSTGRES_PASSWORD: '$DB_PASSWORD'" + DB_VOLUME=" postgresql-data: + driver: local" +elif [ "$DB_TYPE" == "elasticsearch" ]; then + DB_ENVIRONMENT=" DB_TYPE: 'elasticsearch' + DB_URL: 'http://elasticsearch:9200' + DB_USERNAME: 'elastic' + DB_PASSWORD: '$DB_PASSWORD'" + DB_DEPENDS_ON=" depends_on: + - elasticsearch" + DB_SERVICE=" elasticsearch: + image: elasticsearch:8.5.1 + container_name: elasticsearch + restart: on-failure + networks: + - ocean_network + volumes: + - elasticsearch-data:/usr/share/elasticsearch/data + environment: + ES_JAVA_OPTS: '-Xms512m -Xmx512m' + discovery.type: 'single-node' + ELASTIC_PASSWORD: '$DB_PASSWORD' + xpack.security.enabled: 'true' + xpack.security.http.ssl.enabled: 'false'" + DB_VOLUME=" elasticsearch-data: + driver: local" +fi + cat < docker-compose.yml services: ocean-node: @@ -694,7 +777,10 @@ services: environment: PRIVATE_KEY: '$PRIVATE_KEY' RPCS: '{"1":{"rpc":"https://ethereum-rpc.publicnode.com","fallbackRPCs":["https://rpc.ankr.com/eth","https://1rpc.io/eth","https://eth.api.onfinality.io/public"],"chainId":1,"network":"mainnet","chunkSize":100},"10":{"rpc":"https://mainnet.optimism.io","fallbackRPCs":["https://optimism-mainnet.public.blastapi.io","https://rpc.ankr.com/optimism","https://optimism-rpc.publicnode.com"],"chainId":10,"network":"optimism","chunkSize":100},"137":{"rpc":"https://polygon-rpc.com/","fallbackRPCs":["https://polygon-mainnet.public.blastapi.io","https://1rpc.io/matic","https://rpc.ankr.com/polygon"],"chainId":137,"network":"polygon","chunkSize":100},"23294":{"rpc":"https://sapphire.oasis.io","fallbackRPCs":["https://1rpc.io/oasis/sapphire"],"chainId":23294,"network":"sapphire","chunkSize":100},"23295":{"rpc":"https://testnet.sapphire.oasis.io","chainId":23295,"network":"sapphire-testnet","chunkSize":100},"11155111":{"rpc":"https://eth-sepolia.public.blastapi.io","fallbackRPCs":["https://1rpc.io/sepolia","https://eth-sepolia.g.alchemy.com/v2/demo"],"chainId":11155111,"network":"sepolia","chunkSize":100},"11155420":{"rpc":"https://sepolia.optimism.io","fallbackRPCs":["https://endpoints.omniatech.io/v1/op/sepolia/public","https://optimism-sepolia.blockpi.network/v1/rpc/public"],"chainId":11155420,"network":"optimism-sepolia","chunkSize":100}}' - DB_URL: 'http://typesense:8108/?apiKey=xyz' +$DB_ENVIRONMENT + # the node metrics history is kept for this id; a container gets a new hostname + # each time it is made again + INSTANCE_ID: 'ocean-node' IPFS_GATEWAY: 'https://ipfs.io/' ARWEAVE_GATEWAY: 'https://arweave.net/' # LOAD_INITIAL_DDOS: '' @@ -769,23 +855,12 @@ $( options: max-size: "50m" max-file: "5" - depends_on: - - typesense +$DB_DEPENDS_ON - typesense: - image: typesense/typesense:26.0 - container_name: typesense - ports: - - "8108:8108" - networks: - - ocean_network - volumes: - - typesense-data:/data - command: '--data-dir /data --api-key=xyz' +$DB_SERVICE volumes: - typesense-data: - driver: local +$DB_VOLUME node-sqlite: driver: local @@ -814,7 +889,20 @@ echo -e "\e[1;32mP2P IPv6 WebSocket Port: $P2P_ipV6BindWsPort\e[0m" echo "" echo -e "\e[1;32m4)\e[0m Node logs (info and above, 30 days) are written to \e[1;32m./logs\e[0m on this host. Set LOG_FILES_LEVEL=debug in docker-compose.yml for more detail." echo "" -echo -e "\e[1;32m5)\e[0m If using SSL/TLS with a custom domain name, make sure to listen on host port 443 for the HTTP API, or use a reverse proxy with TLS offloading" +if [ -n "$PREVIOUS_DB_TYPE" ] && [ "$PREVIOUS_DB_TYPE" != "$DB_TYPE" ]; then + echo -e "\e[1;31mWarning:\e[0m the database of this folder was \e[1;31m$PREVIOUS_DB_TYPE\e[0m and is now \e[1;31m$DB_TYPE\e[0m. The node state (jobs, services, nonces, auth tokens, buckets) does not move: it stays in the Docker volume of $PREVIOUS_DB_TYPE. A database volume keeps the password of its first start: remove the old volume before you select that database again." + if [ "$PREVIOUS_DB_TYPE" == "sqlite" ]; then + echo -e "The node starts with empty node state on $DB_TYPE: with no nonces, a signed request that the node accepted before can be accepted again, and the paid jobs in the SQLite files are not claimed. The SQLite files stay unused in the Docker volume \e[1;31mnode-sqlite\e[0m. Keep the volume: it also holds the P2P identity store (p2p-store)." + fi + echo "" +fi +if [ "$DB_TYPE" == "sqlite" ]; then + echo -e "\e[1;32m5)\e[0m The node keeps its data in SQLite files, in the Docker volume \e[1;32mnode-sqlite\e[0m. One volume is for one node instance. This script makes one node for each folder. Several node instances can share a PostgreSQL or Elasticsearch database: see docs/database.md." +else + echo -e "\e[1;32m5)\e[0m The $DB_TYPE database has no published port: only the node can reach it. The database password in docker-compose.yml is random and gives full access: keep the file private. To use your own password, change it for the node and for the database \e[1;31mbefore the first start\e[0m: the database reads it only when it creates its data." +fi +echo "" +echo -e "\e[1;32m6)\e[0m If using SSL/TLS with a custom domain name, make sure to listen on host port 443 for the HTTP API, or use a reverse proxy with TLS offloading" echo "" echo -e "If your node is not reachable by other peers (NAT, no public IP, port forwarding issues)," echo -e "refer to the networking guide for help with Dynamic DNS, port forwarding, and circuit relay:" diff --git a/src/@types/DDO/IDdoStateQuery.ts b/src/@types/DDO/IDdoStateQuery.ts deleted file mode 100644 index c85e26b3c..000000000 --- a/src/@types/DDO/IDdoStateQuery.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface IDdoStateQuery { - buildQuery(did?: string, nft?: string, txId?: string): Record -} diff --git a/src/@types/DDO/IMetadataQuery.ts b/src/@types/DDO/IMetadataQuery.ts deleted file mode 100644 index 21cab42ef..000000000 --- a/src/@types/DDO/IMetadataQuery.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { SearchQuery } from './SearchQuery' - -export interface IMetadataQuery { - buildQuery(query: SearchQuery): Record -} diff --git a/src/@types/DDO/SearchQuery.ts b/src/@types/DDO/SearchQuery.ts deleted file mode 100644 index 8fe5a7175..000000000 --- a/src/@types/DDO/SearchQuery.ts +++ /dev/null @@ -1,42 +0,0 @@ -export interface FilterTerm { - term?: { [key: string]: string | number | boolean } - terms?: { [key: string]: (string | number | boolean)[] } - range?: { [key: string]: { gte?: number; lte?: number } } - bool?: any - exists?: { field: string } - match?: { [key: string]: string | number } -} - -export interface BoolQuery { - bool: { - must?: FilterTerm[] - must_not?: (FilterTerm | false | null | undefined)[] - should?: FilterTerm[] - filter?: FilterTerm[] - } -} - -export interface SearchQuery { - q?: string - filter_by?: any - num_hits?: number - start?: number - sort_by?: string - from?: number - size?: number - query?: any - sort?: { [jsonPath: string]: string } - aggs?: any -} - -export interface BaseQueryParams { - esPaginationOptions?: { - from?: number - size?: number - } - nestedQuery?: Partial - filters?: FilterTerm[] - chainIds?: number[] - ignorePurgatory?: boolean - ignoreState?: boolean -} diff --git a/src/@types/OceanNode.ts b/src/@types/OceanNode.ts index e96bdd0b7..3a9efc267 100644 --- a/src/@types/OceanNode.ts +++ b/src/@types/OceanNode.ts @@ -2,16 +2,20 @@ import { Stream } from 'stream' import { RPCS } from './blockchain' import { C2DClusterInfo, C2DDockerConfig } from './C2D/C2D' import { FeeStrategy } from './Fees' -import { Schema } from '../components/database' +import type { DdoSchema } from '../components/database/index.js' import { KeyProviderType } from './KeyManager' import type { PersistentStorageConfig } from './PersistentStorage.js' import type { AccessList } from './AccessList' +export const DATABASE_TYPES = ['sqlite', 'postgresql', 'elasticsearch'] as const +export type DatabaseType = (typeof DATABASE_TYPES)[number] + +// url, username and password are set for postgresql and elasticsearch only export interface OceanNodeDBConfig { - url: string | null + dbType: DatabaseType + url?: string username?: string password?: string - dbType: string | null } // deny list of peer ids and ips (for rate limiting purposes) @@ -167,7 +171,7 @@ export interface OceanNodeConfig { p2pConfig: OceanNodeP2PConfig | null hasIndexer: boolean hasHttp: boolean - dbConfig?: OceanNodeDBConfig + dbConfig: OceanNodeDBConfig // startup database-init retry: attempts, initial backoff (ms) and backoff ceiling (ms) dbInitMaxAttempts: number dbInitRetryDelay: number @@ -218,6 +222,8 @@ export interface OceanNodeConfig { isBootstrap?: boolean validateUnsignedDDO?: boolean jwtSecret?: string + // keys per-instance state (node metrics); getInstanceId() falls back to os.hostname() + instanceId?: string httpCertPath?: string httpKeyPath?: string enableBenchmark?: boolean @@ -309,7 +315,7 @@ export interface OceanNodeStatus { subsidyProviderAllowedImages?: Record // detailed information c2dClusters?: any[] - supportedSchemas?: Schema[] + supportedSchemas?: DdoSchema[] persistentStorage?: { accessLists?: AccessList[] allowBucketSharing?: boolean diff --git a/src/@types/Typesense.ts b/src/@types/Typesense.ts deleted file mode 100644 index 4939374eb..000000000 --- a/src/@types/Typesense.ts +++ /dev/null @@ -1,149 +0,0 @@ -export interface TypesenseAbstractLogger { - error?: any - warn?: any - info?: any - debug?: any - trace?: any -} - -export interface TypesenseNode { - host: string - port: string - protocol: string -} - -export interface TypesenseConfigOptions { - apiKey: string - nodes: TypesenseNode[] - numRetries?: number - retryIntervalSeconds?: number - connectionTimeoutSeconds?: number - logLevel?: string - logger?: TypesenseAbstractLogger -} - -export type TypesenseFieldType = - | 'string' - | 'int32' - | 'int64' - | 'float' - | 'bool' - | 'geopoint' - | 'geopoint[]' - | 'string[]' - | 'int32[]' - | 'int64[]' - | 'float[]' - | 'bool[]' - | 'object' - | 'object[]' - | 'auto' - | 'string*' - -export interface TypesenseCollectionFieldSchema { - name: string - type: TypesenseFieldType - optional?: boolean - facet?: boolean - index?: boolean - sort?: boolean - locale?: string - infix?: boolean - num_dim?: number - [t: string]: unknown -} - -export interface TypesenseCollectionCreateSchema { - name: string - enable_nested_fields?: boolean - fields?: TypesenseCollectionFieldSchema[] -} - -export interface TypesenseCollectionSchema extends TypesenseCollectionCreateSchema { - created_at: number - num_documents: number - num_memory_shards: number -} - -export interface TypesenseCollectionDropFieldSchema { - name: string - drop: true -} - -export interface TypesenseCollectionUpdateSchema extends Partial< - Omit -> { - fields?: (TypesenseCollectionFieldSchema | TypesenseCollectionDropFieldSchema)[] -} - -export type TypesenseDocumentSchema = Record - -type TypesenseOperationMode = 'off' | 'always' | 'fallback' - -export interface TypesenseSearchParams { - [key: string]: any - // From https://typesense.org/docs/latest/api/documents.html#arguments - q: string - query_by: string | string[] - query_by_weights?: string | number[] - prefix?: string | boolean | boolean[] // default: true - filter_by?: string - sort_by?: string | string[] // default: text match desc - facet_by?: string | string[] - max_facet_values?: number - facet_query?: string - facet_query_num_typos?: number - page?: number // default: 1 - per_page?: number // default: 10, max 250 - group_by?: string | string[] - group_limit?: number // default: - include_fields?: string | string[] - exclude_fields?: string | string[] - highlight_fields?: string | string[] // default: all queried fields - highlight_full_fields?: string | string[] // default: all fields - highlight_affix_num_tokens?: number // default: 4 - highlight_start_tag?: string // default: - highlight_end_tag?: string // default: - snippet_threshold?: number // default: 30 - num_typos?: string | number | number[] // default: 2 - min_len_1typo?: number - min_len_2typo?: number - split_join_tokens?: TypesenseOperationMode - exhaustive_search?: boolean - drop_tokens_threshold?: number // default: 10 - typo_tokens_threshold?: number // default: 100 - pinned_hits?: string | string[] - hidden_hits?: string | string[] - limit_hits?: number // default: no limit - pre_segmented_query?: boolean - enable_overrides?: boolean - prioritize_exact_match?: boolean // default: true - prioritize_token_position?: boolean - search_cutoff_ms?: number - use_cache?: boolean - max_candidates?: number - infix?: TypesenseOperationMode | TypesenseOperationMode[] - preset?: string - text_match_type?: 'max_score' | 'max_weight' - vector_query?: string - 'x-typesense-api-key'?: string - 'x-typesense-user-id'?: string - offset?: number - limit?: number -} - -export interface TypesenseSearchResponse { - facet_counts?: any[] - found: number - found_docs?: number - out_of: number - page: number - request_params: any - search_time_ms: number - hits?: any[] - grouped_hits?: { - group_key: string[] - hits: any[] - found?: number - }[] -} diff --git a/src/@types/index.ts b/src/@types/index.ts index 8ac35eeb1..2271a1292 100644 --- a/src/@types/index.ts +++ b/src/@types/index.ts @@ -1,3 +1,2 @@ export * from './OceanNode' export * from './C2D/C2D' -export * from './Typesense' diff --git a/src/OceanNode.ts b/src/OceanNode.ts index 5bdb0f4d0..0fcb8f6fc 100644 --- a/src/OceanNode.ts +++ b/src/OceanNode.ts @@ -1,3 +1,4 @@ +import os from 'os' import { OceanP2P } from './components/P2P/index.js' import { OceanProvider } from './components/Provider/index.js' import { OceanIndexer } from './components/Indexer/index.js' @@ -57,12 +58,14 @@ export class OceanNode { private persistentStorage: PersistentStorageFactory private database: Database // In-memory cache of the maintenance (drain) flag so the hot path (every compute/service - // start) does not hit SQLite per request. Source of truth is the SQLite config DB; this is + // start) does not hit the DB per request. Source of truth is the config DB; this is // loaded from it at startup (loadMaintenanceMode) and kept in sync on every toggle. private maintenanceMode = false // Serializes maintenance toggles so a DB write and its cache update are atomic relative to // any concurrent toggle (prevents DB/cache divergence under overlapping calls). private maintenanceWriteLock: Promise = Promise.resolve() + // HA instances share one PRIVATE_KEY (so one peer id); per-instance state is keyed by this + private instanceId: string private constructor( private config: OceanNodeConfig, @@ -79,8 +82,9 @@ export class OceanNode { this.requestMap = new Map() this.config = config this.database = db + this.instanceId = this.config?.instanceId || os.hostname() - if (this.db && this.db?.authToken) { + if (this.db?.authToken) { this.auth = new Auth(this.db.authToken, config) } if (node) { @@ -97,7 +101,7 @@ export class OceanNode { OCEAN_NODE_LOGGER.info( `Persistent storage enabled (type: ${this.config.persistentStorage.type})` ) - this.persistentStorage = createPersistentStorage(this) + this.persistentStorage = createPersistentStorage(this, this.db?.buckets) } else { OCEAN_NODE_LOGGER.info('Persistent storage disabled') this.persistentStorage = null @@ -301,6 +305,10 @@ export class OceanNode { return this.config } + public getInstanceId(): string { + return this.instanceId + } + /** * Whether the node is in maintenance (drain) mode. Hot-path read of the in-memory cache. */ @@ -309,7 +317,7 @@ export class OceanNode { } /** - * Toggle maintenance mode. Persists the flag to the SQLite config DB (durable, sticky across + * Toggle maintenance mode. Persists the flag to the config DB (durable, sticky across * restarts) and only THEN updates the in-memory cache, so a failed write never leaves the * cache claiming a drain that was not persisted (it would silently revert on reboot). Calls * are serialized on a promise chain so two overlapping toggles cannot end with the DB and the @@ -325,10 +333,10 @@ export class OceanNode { private async applyMaintenanceMode(enabled: boolean): Promise { const db = await this.getDatabase() - if (db?.sqliteConfig) { + if (db?.config) { // createOrUpdateConfig swallows DB errors and returns null; treat that as a failure and // do NOT update the cache, so the node never reports an un-persisted drain. - const result = await db.sqliteConfig.createOrUpdateConfig( + const result = await db.config.createOrUpdateConfig( MAINTENANCE_MODE_CONFIG_KEY, enabled ? 'true' : 'false' ) @@ -352,7 +360,7 @@ export class OceanNode { public async loadMaintenanceMode(): Promise { const envValue = process.env.MAINTENANCE_MODE const db = await this.getDatabase() - const sqliteConfig = db?.sqliteConfig + const configDb = db?.config // Env override wins at boot and is written through to the DB (per design). Only the exact // strings "true"/"false" are honored; any other non-empty value (a typo, "1", "yes") is // IGNORED with a warning, so a leftover malformed env var cannot silently clear a drain. @@ -365,8 +373,8 @@ export class OceanNode { } else { const enabled = normalized === 'true' this.maintenanceMode = enabled - if (sqliteConfig) { - const result = await sqliteConfig.createOrUpdateConfig( + if (configDb) { + const result = await configDb.createOrUpdateConfig( MAINTENANCE_MODE_CONFIG_KEY, enabled ? 'true' : 'false' ) @@ -382,12 +390,12 @@ export class OceanNode { return } } - if (!sqliteConfig) { + if (!configDb) { return } try { - const stored = await sqliteConfig.retrieveValue(MAINTENANCE_MODE_CONFIG_KEY) - // Distinguish a FAILED query from a confirmed-absent key: SQLLiteConfigDatabase + const stored = await configDb.retrieveValue(MAINTENANCE_MODE_CONFIG_KEY) + // Distinguish a FAILED query from a confirmed-absent key: ConfigDatabase // .retrieveValue returns null on a DB error, but { value: null } when the key simply // isn't there (never toggled). if (stored === null) { @@ -529,13 +537,10 @@ export class OceanNode { async getDatabase(forceReload: boolean = false): Promise { if (!this.database || forceReload) { if (!this.dbInitPromise || forceReload) { - const { dbConfig } = this.config - if (dbConfig && dbConfig.url) { - this.dbInitPromise = Database.init(dbConfig).then((db) => { - this.database = db - return db - }) - } + this.dbInitPromise = Database.init(this.config.dbConfig).then((db) => { + this.database = db + return db + }) } return await this.dbInitPromise } diff --git a/src/components/Auth/index.ts b/src/components/Auth/index.ts index bc69e9fa2..9d40bf0f4 100644 --- a/src/components/Auth/index.ts +++ b/src/components/Auth/index.ts @@ -205,10 +205,9 @@ export class Auth { } const createdNum = Number(createdAt) return { - token, address, - created: Number.isFinite(createdNum) ? new Date(createdNum) : new Date(), - validUntil: new Date(validUntilMs), + createdAt: Number.isFinite(createdNum) ? createdNum : Date.now(), + validUntil: validUntilMs, isValid: true } } diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index d98210f63..2e8b74b09 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -34,11 +34,11 @@ import { BlockchainRegistry } from '../BlockchainRegistry/index.js' import { CommandStatus, JobStatus } from '../../@types/commands.js' import { buildJobIdentifier, getDeployedContractBlock } from './utils.js' import { create256Hash } from '../../utils/crypt.js' -import { isReachableConnection } from '../../utils/database.js' +import { DatabaseFactory } from '../database/DatabaseFactory.js' import { sleep } from '../../utils/util.js' import { isReindexingNeeded } from './version.js' import { getPackageVersion } from '../../utils/version.js' -import { DB_EVENTS, ES_CONNECTION_EVENTS } from '../database/ElasticsearchConfigHelper.js' +import { DB_CONNECTION_EVENTS, DB_EVENTS } from '../database/connectionEvents.js' import { OceanNodeConfig } from '../../@types/OceanNode.js' import { clearEventProcessorCache } from './processor.js' import { OceanNode } from '../../OceanNode.js' @@ -60,8 +60,21 @@ let INDEXING_QUEUE: ReindexTask[] = [] const JOBS_QUEUE: JobStatus[] = [] const MAX_CRAWL_RETRIES = 10 +// read and write of the stored node version, for the reindex check +const VERSION_DB_ATTEMPTS = 3 +const VERSION_DB_RETRY_MS = 2000 let numCrawlAttempts = 0 +// The config store returns null when a call fails: try again before giving up. +async function withVersionDbRetry(call: () => Promise): Promise { + let result = await call() + for (let attempt = 1; !result && attempt < VERSION_DB_ATTEMPTS; attempt++) { + await sleep(VERSION_DB_RETRY_MS) + result = await call() + } + return result +} + /** * OceanIndexer - Multi-chain blockchain event indexer * @@ -112,13 +125,13 @@ export class OceanIndexer { } /** - * Listen for Elasticsearch connection events. + * Listen for database connection events. * * CONNECTION_LOST → cancel any pending restart, stop all indexers once. * CONNECTION_RESTORED → debounce restart by 5 s so rapid LOST/RESTORED cycles are a single restart. */ private setupDbConnectionListeners(): void { - ES_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_LOST, async () => { + DB_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_LOST, async () => { if (this.reconnectTimer) { clearTimeout(this.reconnectTimer) this.reconnectTimer = null @@ -135,7 +148,7 @@ export class OceanIndexer { await this.stopAllChainIndexers() }) - ES_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_RESTORED, () => { + DB_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_RESTORED, () => { if (this.isDbConnected) { return } @@ -228,7 +241,7 @@ export class OceanIndexer { const retryInterval = Math.max(blockchain.getKnownRPCs().length * 3000, interval) // give 2 secs per each one const result = await this.startCrawler(blockchain) const dbActive = this.getDatabase() - if (!dbActive || !(await isReachableConnection(dbActive.getConfig().url))) { + if (!dbActive || !(await DatabaseFactory.isReachable(dbActive.getConfig()))) { INDEXER_LOGGER.error(`Giving up start crawling. DB is not online!`) return false } @@ -539,11 +552,21 @@ export class OceanIndexer { public async checkAndTriggerReindexing(): Promise { const currentVersion = getPackageVersion() const dbActive = this.getDatabase() - if (!dbActive || !(await isReachableConnection(dbActive.getConfig().url))) { + if (!dbActive || !(await DatabaseFactory.isReachable(dbActive.getConfig()))) { INDEXER_LOGGER.error(`Giving up reindexing. DB is not online!`) return } - const dbVersion = await dbActive.sqliteConfig?.retrieveValue() + // null means the read failed, which is not the same as "no version stored": try again + // before giving up, or a needed reindex would wait for the next restart + const dbVersion = await withVersionDbRetry(() => + dbActive.config?.retrieveValue('version') + ) + if (!dbVersion) { + INDEXER_LOGGER.error( + `Giving up reindexing. Could not read the node version after ${VERSION_DB_ATTEMPTS} attempts!` + ) + return + } INDEXER_LOGGER.debug( `Node version check: Current=${currentVersion}, DB=${ dbVersion?.value ?? 'not set' @@ -592,8 +615,16 @@ export class OceanIndexer { ) } } - await dbActive.sqliteConfig?.createOrUpdateConfig('version', currentVersion) - INDEXER_LOGGER.info(`Updated node version in database to ${currentVersion}`) + const stored = await withVersionDbRetry(() => + dbActive.config?.createOrUpdateConfig('version', currentVersion) + ) + if (stored) { + INDEXER_LOGGER.info(`Updated node version in database to ${currentVersion}`) + } else { + INDEXER_LOGGER.error( + `Could not store node version ${currentVersion}: the reindex runs again at the next start` + ) + } } else { INDEXER_LOGGER.debug('No reindexing needed based on version check') } diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index bddae5de3..a0d469338 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -2528,6 +2528,12 @@ export class C2DEngineDocker extends C2DEngine { } // No active build abort → orphan (the process running build/pull died). + // The job list comes from a search, which can be older than the last write of + // pullImage/buildImage. A real-time read tells if the job has already moved on. + const [current] = await this.db.getJob(job.jobId) + if (current && current.status !== job.status) { + return + } const isBuild = job.status === C2DStatusNumber.BuildImage const nowSec = Date.now() / 1000 job.status = isBuild @@ -6074,10 +6080,9 @@ export class C2DEngineDocker extends C2DEngine { SERVICE_LOCK_STALE_MS ) } catch (e: any) { - // Fail CLOSED: proceeding without the SQLite lease would disable cross-process - // exclusion exactly when it matters most — SQLITE_BUSY-style errors are most - // likely precisely when two processes contend. Deferring costs little: every - // lifecycle operation writes to the same SQLite file moments later anyway, so it + // Fail CLOSED: proceeding without the DB lease would disable cross-process + // exclusion exactly when it matters most. Deferring costs little: every + // lifecycle operation writes to the same database moments later anyway, so it // could not complete with a broken DB regardless. Callers reject with "retry // shortly" (handlers) or retry on the next tick (the loop). CORE_LOGGER.error( @@ -6125,9 +6130,10 @@ export class C2DEngineDocker extends C2DEngine { } // Releases both lock layers. Never throws: a failed DB delete only leaves a row that - // expires via SERVICE_LOCK_STALE_MS. + // expires via SERVICE_LOCK_STALE_MS. The in-memory layer goes last: while the DB release + // runs, this engine must not take the same lock again, or the release (which matches + // the holder, not one acquisition) could delete the new lock. private async releaseServiceLifecycleLock(serviceId: string): Promise { - this.serviceOpsInFlight.delete(serviceId) try { await this.db.releaseServiceLock(serviceId, this.serviceLockHolderId) CORE_LOGGER.debug( @@ -6135,6 +6141,8 @@ export class C2DEngineDocker extends C2DEngine { ) } catch (e: any) { CORE_LOGGER.error(`service lock DB release failed for ${serviceId}: ${e.message}`) + } finally { + this.serviceOpsInFlight.delete(serviceId) } } diff --git a/src/components/core/handler/escrowHandler.ts b/src/components/core/handler/escrowHandler.ts index cc4288efa..082458969 100644 --- a/src/components/core/handler/escrowHandler.ts +++ b/src/components/core/handler/escrowHandler.ts @@ -10,7 +10,7 @@ import { CORE_LOGGER } from '../../../utils/logging/common.js' import { ESCROW_EVENTS, MAX_ESCROW_CLIENT_SCAN, - TYPESENSE_HITS_CAP + ESCROW_SCAN_PAGE_SIZE } from '../../../utils/constants.js' import { validateEscrowRanges, @@ -118,7 +118,7 @@ export class EscrowEventsHandler extends CommandHandler { const wantOffset = offset && offset > 0 ? offset : 0 const wantSize = size && size > 0 ? size : 100 const needed = wantOffset + wantSize - const pageSize = TYPESENSE_HITS_CAP + const pageSize = ESCROW_SCAN_PAGE_SIZE const matched: Record[] = [] let scanned = 0 diff --git a/src/components/core/handler/nodeMetrics.ts b/src/components/core/handler/nodeMetrics.ts index 0df86f2df..d33c66688 100644 --- a/src/components/core/handler/nodeMetrics.ts +++ b/src/components/core/handler/nodeMetrics.ts @@ -16,9 +16,8 @@ import { getMetricsRetentionDays, isNodeMetricsHistoryEnabled } from '../../../utils/nodeMetricsConfig.js' -import { floorToHour } from '../../database/sqliteNodeMetrics.js' +import { HOUR_MS, floorToHour } from '../../database/nodeMetricsAggregate.js' -const HOUR_MS = 60 * 60 * 1000 // Hard ceiling on returned buckets, independent of the range asked for. 180 days of hourly rows // is 4320; keeping some headroom above the retention window bounds a single response. const MAX_HISTORY_ROWS = 10000 @@ -138,7 +137,13 @@ export class GetNodeMetricsHistoryHandler extends CommandHandler { // coherent and getHourly returns an empty (not malformed) result. if (start > stop) start = stop - const buckets = db.nodeMetrics.getHourly(start, stop, MAX_HISTORY_ROWS) + const instanceId = this.getOceanNode().getInstanceId() + const buckets = await db.nodeMetrics.getHourly( + instanceId, + start, + stop, + MAX_HISTORY_ROWS + ) // Append the live, not-yet-rolled-up current hour (averaged from raw samples) when it falls // in range and the roll-up has not already stored it. Lets a caller see fresh data without @@ -152,7 +157,7 @@ export class GetNodeMetricsHistoryHandler extends CommandHandler { currentHour <= stop && buckets.length < MAX_HISTORY_ROWS ) { - const partial = db.nodeMetrics.getPartialHour(currentHour) + const partial = await db.nodeMetrics.getPartialHour(instanceId, currentHour) if (partial) buckets.push(partial) } diff --git a/src/components/core/handler/queryHandler.ts b/src/components/core/handler/queryHandler.ts index 5f05ceb6a..9f7fecfcc 100644 --- a/src/components/core/handler/queryHandler.ts +++ b/src/components/core/handler/queryHandler.ts @@ -9,6 +9,33 @@ import { } from '../../httpRoutes/validateCommands.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' import { safeStringify } from '../../../utils/logging/redact.js' +import type { Database } from '../../database/index.js' +import { DdoStateKey, Found, QueryError } from '../../database/BaseDatabase.js' +import { + QueryRequest, + assetConditions, + pageOf, + parseQueryRequest, + sortOf +} from '../../database/assetQuery.js' + +// The answer is { results, totalResults } for each type of query. +async function runQuery(database: Database, request: QueryRequest): Promise> { + switch (request.type) { + case 'orders': + return await database.order.find(request.filter, pageOf(request)) + case 'tags': { + const tags = await database.ddo.tags(assetConditions(request.filter), request.size) + return { results: tags, totalResults: tags.length } + } + default: + return await database.ddo.find( + assetConditions(request.filter), + sortOf(request.sort), + pageOf(request) + ) + } +} export class QueryHandler extends CommandHandler { validate(command: QueryCommand): ValidateParams { @@ -21,6 +48,7 @@ export class QueryHandler extends CommandHandler { return validationResponse } try { + const request = parseQueryRequest(task.query) const database = await this.getOceanNode().getDatabase() if (!database || !database.ddo) { CORE_LOGGER.debug('DDO database is not available') @@ -29,15 +57,15 @@ export class QueryHandler extends CommandHandler { status: { httpStatus: 503, error: 'DDO database is not available' } } } - let result = await database.ddo.search(task.query) - if (!result) { - result = [] - } + const result = await runQuery(database, request) return { stream: Readable.from(JSON.stringify(result)), status: { httpStatus: 200 } } } catch (error) { + if (error instanceof QueryError) { + return { stream: null, status: { httpStatus: 400, error: error.message } } + } CORE_LOGGER.error(`Error in QueryHandler: ${error.message}`) return { stream: null, @@ -47,6 +75,7 @@ export class QueryHandler extends CommandHandler { } } +// `query` is the key of the state: { did }, { nft } or { txId } export class QueryDdoStateHandler extends QueryHandler { async handle(task: QueryCommand): Promise { const validation = this.validate(task) @@ -63,7 +92,7 @@ export class QueryDdoStateHandler extends QueryHandler { } } - const result = await database.ddoState.search(task.query) + const result = await database.ddoState.find(task.query as DdoStateKey) CORE_LOGGER.debug(`DDO State search result: ${safeStringify(result)}`) diff --git a/src/components/core/utils/statusHandler.ts b/src/components/core/utils/statusHandler.ts index 90c5c989b..4dd8ca01d 100644 --- a/src/components/core/utils/statusHandler.ts +++ b/src/components/core/utils/statusHandler.ts @@ -8,7 +8,6 @@ import { } from '../../../@types/OceanNode.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' import { OceanNode } from '../../../OceanNode.js' -import { typesenseSchemas } from '../../database/TypesenseSchemas.js' import { SupportedNetwork } from '../../../@types/blockchain.js' import { humanizeHex } from '../../../utils/humanHash.js' import { getPackageVersion } from '../../../utils/version.js' @@ -193,7 +192,7 @@ export async function status( `Error getting c2d clusters: ${error.message}` ) } - nodeStatus.supportedSchemas = typesenseSchemas.ddoSchemas + nodeStatus.supportedSchemas = (await oceanNode.getDatabase())?.ddo?.getSchemas() // Drain-visibility counts: how much already-committed work is still running, so an admin // can tell when the node is drained and safe to reboot. Wrapped in try/catch + throttled diff --git a/src/components/database/AuthTokenDatabase.ts b/src/components/database/AuthTokenDatabase.ts index bcab83445..4ccbbd9d7 100644 --- a/src/components/database/AuthTokenDatabase.ts +++ b/src/components/database/AuthTokenDatabase.ts @@ -1,31 +1,42 @@ -import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { AbstractDatabase } from './BaseDatabase.js' import { OceanNodeDBConfig } from '../../@types/OceanNode.js' -import { SQLiteAuthToken } from './sqliteAuthToken.js' +import { create256Hash } from '../../utils/crypt.js' export interface AuthToken { - token: string address: string - created: Date - validUntil: Date | null + createdAt: number + validUntil: number | null isValid: boolean chainId?: string | null } -export class AuthTokenDatabase extends AbstractDatabase { - private provider: SQLiteAuthToken +// Unix ms from a number or a date text: a client can send the two, and an older release +// stored what it got. NaN when the value is not a time. +export function toUnixMs(value: number | string): number { + const ms = Number(value) + return Math.trunc(Number.isFinite(ms) ? ms : new Date(value).getTime()) +} - private constructor(config: OceanNodeDBConfig, provider?: SQLiteAuthToken) { - super(config) - this.provider = provider - } +// providers get the sha256 of the token (tokenId), never the token itself +export interface AuthTokenDatabaseProvider { + createToken( + tokenId: string, + address: string, + createdAt: number, + validUntil: number | null, + chainId?: string | null + ): Promise + validateTokenEntry(tokenId: string): Promise + invalidateTokenEntry(tokenId: string): Promise + deleteExpiredTokens(now: number): Promise +} - static async create(config: OceanNodeDBConfig): Promise { - DATABASE_LOGGER.debug('Creating AuthTokenDatabase with SQLite') - // SqliteClient creates the parent directory on construction. - const provider = new SQLiteAuthToken('databases/authTokenDatabase.sqlite') - await provider.createTable() - return new AuthTokenDatabase(config, provider) +export class AuthTokenDatabase extends AbstractDatabase { + constructor( + config: OceanNodeDBConfig, + private provider: AuthTokenDatabaseProvider + ) { + super(config) } async createToken( @@ -35,20 +46,31 @@ export class AuthTokenDatabase extends AbstractDatabase { createdAt: number, chainId?: string | null ): Promise { - await this.provider.createToken(token, address, createdAt, validUntil, chainId) + // the default turns an undefined validUntil into null + const until = validUntil === null ? null : toUnixMs(validUntil) + if (Number.isNaN(until)) { + throw new Error('validUntil must be a time in Unix milliseconds or a date') + } + await this.provider.createToken( + create256Hash(token), + address, + createdAt, + until, + chainId + ) return token } async validateToken(token: string): Promise { - const tokenEntry = await this.provider.validateTokenEntry(token) - if (!tokenEntry) { - return null - } - - return tokenEntry + return await this.provider.validateTokenEntry(create256Hash(token)) } async invalidateToken(token: string): Promise { - await this.provider.invalidateTokenEntry(token) + await this.provider.invalidateTokenEntry(create256Hash(token)) + } + + // removes expired and invalidated tokens; returns how many were deleted + async deleteExpiredTokens(now: number): Promise { + return await this.provider.deleteExpiredTokens(now) } } diff --git a/src/components/database/BaseDatabase.ts b/src/components/database/BaseDatabase.ts index 67b2611bb..69b19cb4e 100644 --- a/src/components/database/BaseDatabase.ts +++ b/src/components/database/BaseDatabase.ts @@ -1,23 +1,80 @@ -import { Schema } from '.' import { OceanNodeDBConfig } from '../../@types' import { AccessListUser } from '../../@types/AccessList.js' import { EscrowEvent } from '../../@types/Escrow.js' import { DATABASE_LOGGER } from '../../utils/logging/common.js' -import { ElasticsearchSchema } from './ElasticSchemas.js' -import { TypesenseSchema } from './TypesenseSchemas.js' + +// a group of DDOs of one version (see ddoSchemas.ts); `index` is its name +export interface DdoSchema { + index: string +} + +// A query that the node cannot run: the caller gets the message (HTTP 400). +export class QueryError extends Error {} + +export const DEFAULT_PAGE_SIZE = 100 +// the last document that one search can reach, as on Elasticsearch (its default window) +export const MAX_RESULT_WINDOW = 10000 + +export type Scalar = string | number | boolean + +// A condition on a stored document. `paths` are the places of one field, one for each +// document version (a DDO v4 path and its v5 `credentialSubject.` path); the condition +// holds when one of them holds. Each backend writes its own fixed query for each kind. +export type Condition = + // a value of the field is one of these + | { paths: string[]; anyOf: Scalar[] } + // no value of the field is one of these (a document without the field passes) + | { paths: string[]; noneOf: Scalar[] } + // a value of the field is in the range + | { paths: string[]; range: { gt?: Scalar; gte?: Scalar; lt?: Scalar; lte?: Scalar } } + // the field, or an object with fields below it, is in the document + | { paths: string[]; exists: true } + // each word is in a value of one of the fields + | { paths: string[]; words: string[] } + // one of these conditions holds + | { either: Condition[] } + +export interface SortSpec { + paths: string[] + order: 'asc' | 'desc' + // a backend with a text score sorts by it first, then by `paths`; the others by `paths` + relevance?: boolean +} + +// `offset` counts documents +export interface Page { + offset: number + limit: number +} + +export interface Found> { + results: T[] + totalResults: number +} + +export interface TagCount { + tag: string + count: number +} + +// one of the three keys of a DDO state +export type DdoStateKey = { did: string } | { nft: string } | { txId: string } + +// the orders of one account +export type OrderOwner = { consumer: string } | { payer: string } export abstract class AbstractDatabase { protected config: OceanNodeDBConfig - protected schema: TypesenseSchema - constructor(config: OceanNodeDBConfig, schema?: TypesenseSchema) { + constructor(config: OceanNodeDBConfig) { this.config = config - this.schema = schema } } export abstract class AbstractNonceDatabase extends AbstractDatabase { abstract create(address: string, nonce: number): Promise abstract retrieve(address: string): Promise + // Writes only a nonce above the stored one, and returns null when it did not: two + // requests that check the same signed nonce at the same time cannot both pass. abstract update(address: string, nonce: number): Promise abstract delete(address: string): Promise @@ -62,6 +119,8 @@ export abstract class AbstractAccessListDatabase extends AbstractDatabase { abstract delete(chainId: number, contractAddress: string): Promise } +const DEFAULT_LOG_RETENTION_MS = 30 * 24 * 60 * 60 * 1000 + export abstract class AbstractLogDatabase extends AbstractDatabase { abstract insertLog(logEntry: Record): Promise abstract retrieveLog(id: string): Promise | null> @@ -77,6 +136,21 @@ export abstract class AbstractLogDatabase extends AbstractDatabase { abstract delete(logId: string): Promise abstract deleteOldLogs(): Promise abstract getLogsCount(): Promise + + // the time that a log stays (LOG_RETENTION_TIME, milliseconds). A value of 0 or less + // would delete the new logs too, and a value so large that the cutoff is not a valid + // date would make the cleanup fail: both get the default. + protected retentionMs(): number { + const raw = process.env.LOG_RETENTION_TIME + const retention = Number(raw) + if (raw === undefined || raw === '') return DEFAULT_LOG_RETENTION_MS + const cutoff = new Date(Date.now() - retention).getTime() + if (retention > 0 && Number.isFinite(cutoff)) return retention + DATABASE_LOGGER.warn( + `Invalid LOG_RETENTION_TIME "${raw}", using ${DEFAULT_LOG_RETENTION_MS} ms` + ) + return DEFAULT_LOG_RETENTION_MS + } } export abstract class AbstractDdoStateDatabase extends AbstractDatabase { @@ -91,7 +165,8 @@ export abstract class AbstractDdoStateDatabase extends AbstractDatabase { abstract retrieve(did: string): Promise | null> - abstract search(query: Record): Promise + // ten states at most; null on a database error + abstract find(key: DdoStateKey): Promise[] | null> abstract update( chainId: number, @@ -105,20 +180,9 @@ export abstract class AbstractDdoStateDatabase extends AbstractDatabase { abstract delete(did: string): Promise } -export abstract class AbstractOrderDatabase { - protected config: OceanNodeDBConfig - protected schema: Schema - - constructor(config: OceanNodeDBConfig, schema: Schema) { - this.config = config - this.schema = schema - } - - abstract search( - query: Record, - maxResultsPerPage?: number, - pageNumber?: number - ): Promise[] | null> +export abstract class AbstractOrderDatabase extends AbstractDatabase { + // the newest orders first + abstract find(owner: OrderOwner, page: Page): Promise abstract create( orderId: string, @@ -147,15 +211,7 @@ export abstract class AbstractOrderDatabase { abstract delete(orderId: string): Promise } -export abstract class AbstractEscrowDatabase { - protected config: OceanNodeDBConfig - protected schema: Schema - - constructor(config: OceanNodeDBConfig, schema: Schema) { - this.config = config - this.schema = schema - } - +export abstract class AbstractEscrowDatabase extends AbstractDatabase { abstract create(event: EscrowEvent): Promise abstract retrieve(id: string): Promise | null> @@ -169,30 +225,14 @@ export abstract class AbstractEscrowDatabase { abstract delete(id: string): Promise } -export abstract class AbstractDdoDatabase { - protected config: OceanNodeDBConfig - protected schemas: Schema[] - - constructor(config: OceanNodeDBConfig, schemas: Schema[]) { - this.config = config - this.schemas = schemas - } +export abstract class AbstractDdoDatabase extends AbstractDatabase { + abstract getSchemas(): DdoSchema[] - abstract getSchemas(): Schema[] - - public isElasticsearchSchema(schema: Schema): schema is ElasticsearchSchema { - return (schema as ElasticsearchSchema).index !== undefined - } + // the DDOs of all versions, sorted by `sort` and then by id + abstract find(conditions: Condition[], sort: SortSpec[], page: Page): Promise - public isTypesenseSchema(schema: Schema): schema is TypesenseSchema { - return (schema as TypesenseSchema).name !== undefined - } - - abstract search( - query: Record, - maxResultsPerPage?: number, - pageNumber?: number - ): Promise + // the distinct tags of the DDOs that match, the most used first + abstract tags(conditions: Condition[], limit: number): Promise abstract create(ddo: Record): Promise @@ -202,5 +242,5 @@ export abstract class AbstractDdoDatabase { abstract delete(id: string): Promise - abstract deleteAllAssetsFromChain(chainId: number, batchSize?: number): Promise + abstract deleteAllAssetsFromChain(chainId: number): Promise } diff --git a/src/components/database/BucketRegistry.ts b/src/components/database/BucketRegistry.ts new file mode 100644 index 000000000..5f4f85042 --- /dev/null +++ b/src/components/database/BucketRegistry.ts @@ -0,0 +1,29 @@ +export type BucketRow = { + bucketId: string + owner: string + accessListJson: string + createdAt: number + label: string | null +} + +// persistent-storage bucket metadata; the bucket files live in the storage backend +export interface BucketRegistry { + // a conflict updates accessListJson only, so a re-create never clobbers a rename + upsertBucket( + bucketId: string, + owner: string, + accessListJson: string, + createdAt: number, + label: string | null + ): Promise + getBucket(bucketId: string): Promise + // exact owner match, createdAt ascending + listBucketsByOwner(owner: string): Promise + deleteBucket(bucketId: string): Promise + // false when the bucket is unknown or the owner does not match + updateBucketLabel( + bucketId: string, + owner: string, + label: string | null + ): Promise +} diff --git a/src/components/database/C2DDatabase.ts b/src/components/database/C2DDatabase.ts index 3542e01ff..aea47f564 100755 --- a/src/components/database/C2DDatabase.ts +++ b/src/components/database/C2DDatabase.ts @@ -1,5 +1,3 @@ -import path from 'path' -import fs from 'fs' import { ComputeEnvironment, DBComputeJob, @@ -11,36 +9,75 @@ import { ServiceModelDownload, ServiceReadiness } from '../../@types/C2D/ServiceOnDemand.js' -import { SQLiteCompute } from './sqliteCompute.js' import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { OceanNodeDBConfig } from '../../@types/OceanNode.js' -import { TypesenseSchema } from './TypesenseSchemas.js' import { AbstractDatabase } from './BaseDatabase.js' import { OceanNode } from '../../OceanNode.js' import { generateUniqueID } from '../core/compute/utils.js' + +// the lifecycle fields a best-effort service job body patch must still find unchanged +export type ServiceJobExpected = { + owner: string + clusterHash: string + status: number + containerId: string +} + +export interface ComputeDatabaseProvider { + newJob(job: DBComputeJob): Promise + getJob(jobId?: string, agreementId?: string, owner?: string): Promise + updateJob(job: DBComputeJob): Promise + getRunningJobs(engine?: string, environment?: string): Promise + deleteJob(jobId: string): Promise + getFinishedJobs(environments?: string[]): Promise + getJobs( + environments?: string[], + fromTimestamp?: number, + consumerAddrs?: string[], + status?: C2DStatusNumber, + runningJobs?: boolean + ): Promise + getJobsByStatus( + environments: string[], + status: C2DStatusNumber[] + ): Promise + updateImage(image: string): Promise + deleteImage(image: string): Promise + getOldImages(retentionDays: number): Promise + newServiceJob(job: ServiceJob): Promise + getServiceJob(serviceId?: string, owner?: string): Promise + updateServiceJob(job: ServiceJob): Promise + updateServiceJobMetrics( + serviceId: string, + expected: ServiceJobExpected, + runtimeMetrics: ContainerMetricsSnapshot + ): Promise + updateServiceJobReadiness( + serviceId: string, + expected: ServiceJobExpected, + readiness: ServiceReadiness | undefined, + modelDownload?: ServiceModelDownload + ): Promise + getRunningServiceJobs(clusterHash?: string): Promise + getExpiredServiceJobs(clusterHash?: string): Promise + getExpiredServiceJobsBefore( + expiresBefore: number, + clusterHash?: string + ): Promise + getPendingServiceStarts(clusterHash?: string): Promise + acquireServiceLock(serviceId: string, holder: string, staleMs: number): Promise + releaseServiceLock(serviceId: string, holder: string): Promise + refreshServiceLocks(holder: string): Promise + isServiceLocked(serviceId: string, staleMs: number): Promise +} + export class C2DDatabase extends AbstractDatabase { - private provider: SQLiteCompute - - constructor(config: OceanNodeDBConfig, schema: TypesenseSchema) { - super(config, schema) - return (async (): Promise => { - // Fall back to SQLite - DATABASE_LOGGER.debug('Creating C2DDatabase with SQLite') - - // Ensure the directory exists before instantiating SQLiteProvider - const dbDir = path.dirname('databases/c2dDatabase.sqlite') - if (!fs.existsSync(dbDir)) { - fs.mkdirSync(dbDir, { recursive: true }) - } - this.provider = new SQLiteCompute('databases/c2dDatabase.sqlite') - await this.provider.createTable() - await this.provider.createImageTable() - await this.provider.createServiceTable() - await this.provider.createServiceLocksTable() - - return this - })() as unknown as C2DDatabase + constructor( + config: OceanNodeDBConfig, + private provider: ComputeDatabaseProvider + ) { + super(config) } async newJob(job: DBComputeJob): Promise { @@ -97,12 +134,7 @@ export class C2DDatabase extends AbstractDatabase { // only when the row still matches the expected owner/clusterHash/status/containerId. Best-effort. async updateServiceJobMetrics( serviceId: string, - expected: { - owner: string - clusterHash: string - status: number - containerId: string - }, + expected: ServiceJobExpected, runtimeMetrics: ContainerMetricsSnapshot ): Promise { return await this.provider.updateServiceJobMetrics( @@ -114,12 +146,7 @@ export class C2DDatabase extends AbstractDatabase { async updateServiceJobReadiness( serviceId: string, - expected: { - owner: string - clusterHash: string - status: number - containerId: string - }, + expected: ServiceJobExpected, readiness: ServiceReadiness | undefined, modelDownload?: ServiceModelDownload ): Promise { @@ -150,7 +177,7 @@ export class C2DDatabase extends AbstractDatabase { return await this.provider.getPendingServiceStarts(clusterHash) } - // ── Service lifecycle locks (cross-process; see sqliteCompute.ts) ─────── + // ── Service lifecycle locks (cross-process) ───────────────────────────── async acquireServiceLock( serviceId: string, @@ -180,7 +207,11 @@ export class C2DDatabase extends AbstractDatabase { return await this.provider.getFinishedJobs(environments) } - /** @param fromTimestamp lower bound in Unix seconds — see SQLiteCompute.getJobs. */ + /** + * @param fromTimestamp lower bound in Unix **seconds** (the unit dateCreated/dateFinished + * are stored in). Callers normalize before getting here — see parseFromTimestampSeconds + * in core/utils/timestamps.ts. + */ async getJobs( environments?: string[], fromTimestamp?: number, diff --git a/src/components/database/ConfigDatabase.ts b/src/components/database/ConfigDatabase.ts new file mode 100644 index 000000000..cd1b9e098 --- /dev/null +++ b/src/components/database/ConfigDatabase.ts @@ -0,0 +1,9 @@ +// Node-wide key/value settings (indexer version, maintenance mode flag). +// Both methods return null on a DB error; retrieveValue returns { value: null } for an absent key. +export interface ConfigDatabase { + createOrUpdateConfig( + key: string, + value: string + ): Promise<{ key: string; value: string } | null> + retrieveValue(key: string): Promise<{ value: string | null } | null> +} diff --git a/src/components/database/DatabaseFactory.ts b/src/components/database/DatabaseFactory.ts index f5cc288af..88701f7b6 100644 --- a/src/components/database/DatabaseFactory.ts +++ b/src/components/database/DatabaseFactory.ts @@ -1,164 +1,64 @@ -import { OceanNodeDBConfig } from '../../@types' -import { +import type { OceanNodeDBConfig } from '../../@types/OceanNode.js' +import type { AuthTokenDatabase } from './AuthTokenDatabase.js' +import type { AbstractAccessListDatabase, AbstractDdoDatabase, AbstractDdoStateDatabase, AbstractEscrowDatabase, AbstractIndexerDatabase, AbstractLogDatabase, + AbstractNonceDatabase, AbstractOrderDatabase } from './BaseDatabase.js' -import { - ElasticsearchAccessListDatabase, - ElasticsearchDdoDatabase, - ElasticsearchDdoStateDatabase, - ElasticsearchEscrowDatabase, - ElasticsearchIndexerDatabase, - ElasticsearchLogDatabase, - ElasticsearchOrderDatabase -} from './ElasticSearchDatabase.js' -import { typesenseSchemas } from './TypesenseSchemas.js' -import { - TypesenseAccessListDatabase, - TypesenseDdoDatabase, - TypesenseDdoStateDatabase, - TypesenseEscrowDatabase, - TypesenseIndexerDatabase, - TypesenseLogDatabase, - TypesenseOrderDatabase -} from './TypesenseDatabase.js' -import { elasticSchemas } from './ElasticSchemas.js' -import { IDdoStateQuery } from '../../@types/DDO/IDdoStateQuery.js' -import { TypesenseDdoStateQuery } from './TypesenseDdoStateQuery.js' -import { ElasticSearchDdoStateQuery } from './ElasticSearchDdoStateQuery.js' -import { TypesenseMetadataQuery } from './TypesenseMetadataQuery.js' -import { IMetadataQuery } from '../../@types/DDO/IMetadataQuery.js' -import { ElasticSearchMetadataQuery } from './ElasticSearchMetadataQuery.js' -import { DB_TYPES } from '../../utils/index.js' -import { C2DDatabase } from './C2DDatabase.js' -import { SQLLiteNonceDatabase } from './SQLLiteNonceDatabase.js' -import { SQLLiteConfigDatabase } from './SQLLiteConfigDatabase.js' -import { AuthTokenDatabase } from './AuthTokenDatabase.js' -import { NodeMetricsDatabase } from './sqliteNodeMetrics.js' +import type { BucketRegistry } from './BucketRegistry.js' +import type { C2DDatabase } from './C2DDatabase.js' +import type { ConfigDatabase } from './ConfigDatabase.js' +import { createElasticsearchStores } from './ElasticsearchStores.js' +import type { NodeMetricsProvider } from './NodeMetricsDatabase.js' +import { createPostgresStores } from './sql/PostgresStores.js' +import { createSqliteStores } from './sql/SqliteStores.js' + +// all the stores of a node; each backend builds the full set +export interface DatabaseStores { + ddo: AbstractDdoDatabase + nonce: AbstractNonceDatabase + indexer: AbstractIndexerDatabase + logs: AbstractLogDatabase + order: AbstractOrderDatabase + ddoState: AbstractDdoStateDatabase + accessList: AbstractAccessListDatabase + escrow: AbstractEscrowDatabase + config: ConfigDatabase + c2d: C2DDatabase + authToken: AuthTokenDatabase + buckets: BucketRegistry + nodeMetrics: NodeMetricsProvider +} +// the only place that knows which backends exist export class DatabaseFactory { - private static databaseMap = { - elasticsearch: { - ddo: (config: OceanNodeDBConfig) => - new ElasticsearchDdoDatabase(config, elasticSchemas.ddoSchemas), - indexer: (config: OceanNodeDBConfig) => new ElasticsearchIndexerDatabase(config), - log: (config: OceanNodeDBConfig) => new ElasticsearchLogDatabase(config), - order: (config: OceanNodeDBConfig) => - new ElasticsearchOrderDatabase(config, elasticSchemas.orderSchema), - ddoState: (config: OceanNodeDBConfig) => new ElasticsearchDdoStateDatabase(config), - ddoStateQuery: () => new ElasticSearchDdoStateQuery(), - metadataQuery: () => new ElasticSearchMetadataQuery(), - accessList: (config: OceanNodeDBConfig) => - new ElasticsearchAccessListDatabase(config), - escrow: (config: OceanNodeDBConfig) => - new ElasticsearchEscrowDatabase(config, elasticSchemas.escrowSchema) - }, - typesense: { - ddo: (config: OceanNodeDBConfig) => - new TypesenseDdoDatabase(config, typesenseSchemas.ddoSchemas), - indexer: (config: OceanNodeDBConfig) => - new TypesenseIndexerDatabase(config, typesenseSchemas.indexerSchemas), - log: (config: OceanNodeDBConfig) => - new TypesenseLogDatabase(config, typesenseSchemas.logSchemas), - order: (config: OceanNodeDBConfig) => - new TypesenseOrderDatabase(config, typesenseSchemas.orderSchema), - ddoState: (config: OceanNodeDBConfig) => - new TypesenseDdoStateDatabase(config, typesenseSchemas.ddoStateSchema), - ddoStateQuery: () => new TypesenseDdoStateQuery(), - metadataQuery: () => new TypesenseMetadataQuery(), - accessList: (config: OceanNodeDBConfig) => - new TypesenseAccessListDatabase(config, typesenseSchemas.accessListSchema), - escrow: (config: OceanNodeDBConfig) => - new TypesenseEscrowDatabase(config, typesenseSchemas.escrowSchema) + static createStores(config: OceanNodeDBConfig): Promise { + switch (config.dbType) { + case 'sqlite': + return createSqliteStores(config) + case 'postgresql': + return createPostgresStores(config) + case 'elasticsearch': + return createElasticsearchStores(config) + default: + throw new Error(`Unknown database type: ${config.dbType}`) } } - private static getDatabaseType(config: OceanNodeDBConfig): any { - return config.dbType === DB_TYPES.ELASTIC_SEARCH - ? DB_TYPES.ELASTIC_SEARCH - : DB_TYPES.TYPESENSE - } - - private static createDatabase( - databaseType: keyof (typeof DatabaseFactory.databaseMap)['elasticsearch'], - config: OceanNodeDBConfig - ): T { - const dbTypeKey: keyof typeof this.databaseMap = this.getDatabaseType(config) - const databaseCreator = this.databaseMap[dbTypeKey][databaseType] - return databaseCreator(config) as T - } - - static async createNonceDatabase( - config: OceanNodeDBConfig - ): Promise { - return await new SQLLiteNonceDatabase(config, typesenseSchemas.nonceSchemas) - } - - static createDdoDatabase(config: OceanNodeDBConfig): Promise { - return this.createDatabase('ddo', config) - } - - static async createC2DDatabase(config: OceanNodeDBConfig): Promise { - return await new C2DDatabase(config, typesenseSchemas.c2dSchemas) - } - - static async createAuthTokenDatabase( - config: OceanNodeDBConfig - ): Promise { - return await AuthTokenDatabase.create(config) - } - - static createIndexerDatabase( - config: OceanNodeDBConfig - ): Promise { - return this.createDatabase('indexer', config) - } - - static createLogDatabase(config: OceanNodeDBConfig): Promise { - return this.createDatabase('log', config) - } - - static createOrderDatabase(config: OceanNodeDBConfig): Promise { - return this.createDatabase('order', config) - } - - static createDdoStateDatabase( - config: OceanNodeDBConfig - ): Promise { - return this.createDatabase('ddoState', config) - } - - static createDdoStateQuery(config: OceanNodeDBConfig): Promise { - return this.createDatabase('ddoStateQuery', config) - } - - static createMetadataQuery(config: OceanNodeDBConfig): Promise { - return this.createDatabase('metadataQuery', config) - } - - static async createConfigDatabase(): Promise { - return await new SQLLiteConfigDatabase() - } - - // eslint-disable-next-line require-await - static async createNodeMetricsDatabase(): Promise { - return new NodeMetricsDatabase() - } - - static createAccessListDatabase( - config: OceanNodeDBConfig - ): Promise { - return this.createDatabase('accessList', config) - } - - static createEscrowDatabase( - config: OceanNodeDBConfig - ): Promise { - return this.createDatabase('escrow', config) + // a cheap check before createStores; a backend with no such check reports its failed + // connection from createStores + static async isReachable(config: OceanNodeDBConfig): Promise { + if (config.dbType !== 'elasticsearch') return true + try { + await fetch(config.url) + return true + } catch { + return false + } } } diff --git a/src/components/database/ElasticSchemas.ts b/src/components/database/ElasticSchemas.ts index 565a68ab6..36e32a605 100644 --- a/src/components/database/ElasticSchemas.ts +++ b/src/components/database/ElasticSchemas.ts @@ -1,4 +1,5 @@ import { ddoElasticMappings } from '@oceanprotocol/ddo-js' +import { DDO_SHORT_SCHEMA } from './ddoSchemas.js' export type ElasticsearchSchema = { index: string @@ -11,6 +12,7 @@ export type ElasticsearchSchema = { } } } + settings?: { [setting: string]: any } } } @@ -25,10 +27,9 @@ export type ElasticsearchSchemas = { escrowSchema: ElasticsearchSchema } -// "op_ddo_short" is a node-side index for deprecated DDOs (state !== 0). -// Not part of the DDO spec, so it stays here rather than in @oceanprotocol/ddo-js. +// not part of the DDO spec, so its mappings stay here rather than in @oceanprotocol/ddo-js const ddoShortSchema: ElasticsearchSchema = { - index: 'op_ddo_short', + index: DDO_SHORT_SCHEMA, body: { mappings: { properties: { @@ -52,8 +53,36 @@ const ddoShortSchema: ElasticsearchSchema = { } } +// The v5 mapping of @oceanprotocol/ddo-js does not index `services.compute` (enabled: +// false). The DDO search filters on its trusted algorithms, so the node maps these two +// fields; the other compute fields stay unindexed (dynamic: false). Elasticsearch cannot +// change `enabled` on an index that exists: docs/database.md has the upgrade step. +const V5_DDO_SCHEMA = 'op_ddo_v5.0.0' +const searchableCompute = { + type: 'object', + dynamic: false, + properties: { + publisherTrustedAlgorithms: { + type: 'object', + properties: { did: { type: 'keyword' }, serviceId: { type: 'keyword' } } + }, + publisherTrustedAlgorithmPublishers: { type: 'keyword' } + } +} + +function withSearchableCompute(schema: ElasticsearchSchema): ElasticsearchSchema { + if (schema.index !== V5_DDO_SCHEMA) return schema + const copy: any = structuredClone(schema) + copy.body.mappings.properties.credentialSubject.properties.services.properties.compute = + searchableCompute + return copy +} + export const elasticSchemas: ElasticsearchSchemas = { - ddoSchemas: [...(ddoElasticMappings as ElasticsearchSchema[]), ddoShortSchema], + ddoSchemas: [ + ...(ddoElasticMappings as ElasticsearchSchema[]).map(withSearchableCompute), + ddoShortSchema + ], nonceSchemas: { index: 'nonce', body: { @@ -69,6 +98,7 @@ export const elasticSchemas: ElasticsearchSchemas = { body: { mappings: { properties: { + id: { type: 'keyword' }, lastIndexedBlock: { type: 'long' } } } @@ -106,10 +136,12 @@ export const elasticSchemas: ElasticsearchSchemas = { } }, ddoStateSchema: { - index: 'state', + index: 'ddo_state', body: { mappings: { properties: { + id: { type: 'keyword' }, + // some chain ids are above the integer range chainId: { type: 'long' }, did: { type: 'keyword' }, nft: { type: 'keyword' }, @@ -125,7 +157,7 @@ export const elasticSchemas: ElasticsearchSchemas = { body: { mappings: { properties: { - chainId: { type: 'integer' }, + chainId: { type: 'long' }, contractAddress: { type: 'keyword' }, name: { type: 'keyword' }, symbol: { type: 'keyword' }, @@ -180,3 +212,182 @@ export const elasticSchemas: ElasticsearchSchemas = { } } } + +// numeric node metrics fields shared by the samples and hourly indices +const metricsNumericFields = Object.fromEntries( + [ + 'cpuUsagePercent', + 'coresAllocated', + 'hostCores', + 'throttledCount', + 'memUsedBytes', + 'memLimitBytes', + 'hostFreeBytes', + 'hostTotalBytes', + 'diskUsedBytes', + 'rxBytes', + 'txBytes', + 'jobsRunning', + 'jobsRunningFree', + 'jobsQueued', + 'jobsQueuedFree', + 'sampledContainers' + ].map((field) => [field, { type: 'double' }]) +) + +// owner addresses match whatever casing they were written with +const lowercaseNormalizerSettings = { + analysis: { + normalizer: { + lowercase: { type: 'custom', filter: ['lowercase'] } + } + } +} + +// Node state: config, auth tokens, buckets, metrics, C2D jobs, service jobs and locks. +export const stateSchemas: { + nodeConfig: ElasticsearchSchema + authTokens: ElasticsearchSchema + buckets: ElasticsearchSchema + metricsSamples: ElasticsearchSchema + metricsHourly: ElasticsearchSchema + c2dJobs: ElasticsearchSchema + serviceJobs: ElasticsearchSchema + dockerImages: ElasticsearchSchema + serviceLocks: ElasticsearchSchema +} = { + nodeConfig: { + index: 'node_config', + body: { + mappings: { + properties: { + value: { type: 'keyword' } + } + } + } + }, + // id = sha256 of the token; the token itself is never stored + authTokens: { + index: 'auth_tokens', + body: { + mappings: { + properties: { + address: { type: 'keyword' }, + createdAt: { type: 'long' }, + validUntil: { type: 'long' }, + isValid: { type: 'boolean' }, + chainId: { type: 'keyword' } + } + } + } + }, + // persistent-storage bucket registry: id = bucketId + buckets: { + index: 'buckets', + body: { + mappings: { + properties: { + owner: { type: 'keyword' }, + accessListJson: { type: 'text', index: false }, + createdAt: { type: 'long' }, + label: { type: 'keyword' } + } + } + } + }, + // raw minute samples per instance: auto id + metricsSamples: { + index: 'node_metrics_samples', + body: { + mappings: { + properties: { + instanceId: { type: 'keyword' }, + collectedAt: { type: 'long' }, + ...metricsNumericFields, + gpu: { type: 'text', index: false }, + env: { type: 'text', index: false } + } + } + } + }, + // hourly averages per instance: id = `${instanceId}:${hourStart}` + metricsHourly: { + index: 'node_metrics_hourly', + body: { + mappings: { + properties: { + instanceId: { type: 'keyword' }, + hourStart: { type: 'long' }, + sampleCount: { type: 'integer' }, + ...metricsNumericFields, + gpu: { type: 'text', index: false }, + env: { type: 'text', index: false } + } + } + } + }, + // C2D jobs: id = jobId; dates are decimal Unix seconds; expireTimestamp holds maxJobDuration + c2dJobs: { + index: 'c2d_jobs', + body: { + settings: lowercaseNormalizerSettings, + mappings: { + properties: { + owner: { type: 'keyword', normalizer: 'lowercase' }, + did: { type: 'keyword' }, + agreementId: { type: 'keyword' }, + environment: { type: 'keyword' }, + statusText: { type: 'keyword' }, + inputDID: { type: 'keyword' }, + algoDID: { type: 'keyword' }, + status: { type: 'integer' }, + dateCreated: { type: 'double' }, + dateFinished: { type: 'double' }, + expireTimestamp: { type: 'long' }, + body: { type: 'object', enabled: false } + } + } + } + }, + // service-on-demand jobs: id = serviceId; body holds the whole job + serviceJobs: { + index: 'service_jobs', + body: { + settings: lowercaseNormalizerSettings, + mappings: { + properties: { + owner: { type: 'keyword', normalizer: 'lowercase' }, + clusterHash: { type: 'keyword' }, + status: { type: 'integer' }, + expiresAt: { type: 'long' }, + dateCreated: { type: 'keyword' }, + outputArchiveCount: { type: 'integer' }, + body: { type: 'object', enabled: false } + } + } + } + }, + // docker image usage: id = image name; lastUsedTimestamp in Unix seconds + dockerImages: { + index: 'docker_images', + body: { + mappings: { + properties: { + lastUsedTimestamp: { type: 'long' } + } + } + } + }, + // service lifecycle locks: id = serviceId; acquiredAt in Unix ms + serviceLocks: { + index: 'service_locks', + body: { + mappings: { + properties: { + holder: { type: 'keyword' }, + acquiredAt: { type: 'long' } + } + } + } + } +} diff --git a/src/components/database/ElasticSearchDatabase.ts b/src/components/database/ElasticSearchDatabase.ts index bda17bd1a..1c70f0ebe 100644 --- a/src/components/database/ElasticSearchDatabase.ts +++ b/src/components/database/ElasticSearchDatabase.ts @@ -6,63 +6,174 @@ import { AbstractEscrowDatabase, AbstractIndexerDatabase, AbstractLogDatabase, - AbstractOrderDatabase + AbstractNonceDatabase, + AbstractOrderDatabase, + Condition, + DdoStateKey, + Found, + MAX_RESULT_WINDOW, + OrderOwner, + Page, + SortSpec, + TagCount } from './BaseDatabase.js' +import { DDO_FIELDS } from './assetQuery.js' +import { esQuery, esSort, totalOf } from './esConditions.js' import { AccessListUser } from '../../@types/AccessList.js' import { EscrowEvent } from '../../@types/Escrow.js' -import { buildElasticQuery } from './escrowSearch.js' -import { createElasticsearchClientWithRetry } from './ElasticsearchConfigHelper.js' +import { buildElasticQuery, escrowPage } from './escrowSearch.js' +import { + createElasticsearchClientWithRetry, + deleteMatching, + ensureIndex, + getOrNull, + isStatus +} from './ElasticsearchConfigHelper.js' import { OceanNodeDBConfig } from '../../@types' -import { ElasticsearchSchema } from './ElasticSchemas.js' +import { ElasticsearchSchema, elasticSchemas } from './ElasticSchemas.js' +import { addressCasingVariants, normalizeAddress } from '../../utils/evmAddress.js' import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { LOG_LEVELS_STR } from '../../utils/logging/Logger.js' import { safeStringify } from '../../utils/logging/redact.js' import { validateDDO } from '../../utils/asset.js' -import { DDOManager } from '@oceanprotocol/ddo-js' +import { ddoSchemaName } from './ddoSchemas.js' const LOG_INSERT_FAILED_KEY = 'es-log-insert-failed' let logInsertFailing = false -export class ElasticsearchIndexerDatabase extends AbstractIndexerDatabase { - private client: Client - private index: string +// Each store gets the client on each call: the health monitor replaces the client after a +// reconnect, and a client that it closed fails every request. + +// Nonces are read only by real-time `get`, never by search, so writes skip `refresh: 'wait_for'`. +export class ElasticsearchNonceDatabase extends AbstractNonceDatabase { + private readonly index = elasticSchemas.nonceSchemas.index - constructor(config: OceanNodeDBConfig) { + private constructor(config: OceanNodeDBConfig) { super(config) - this.index = 'indexer' + } + + static async create(config: OceanNodeDBConfig): Promise { + await ensureIndex(config, elasticSchemas.nonceSchemas) + return new ElasticsearchNonceDatabase(config) + } - return (async (): Promise => { - this.client = await createElasticsearchClientWithRetry(config) - await this.initializeIndex() - return this - })() as unknown as ElasticsearchIndexerDatabase + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) } - private async initializeIndex() { + private async write(address: string, nonce: number) { + const id = normalizeAddress(address) + const client = await this.client() + await client.index({ + index: this.index, + id, + body: { nonce } + }) + return { id, nonce } + } + + async create(address: string, nonce: number) { try { - const indexExists = await this.client.indices.exists({ index: this.index }) - if (!indexExists) { - await this.client.indices.create({ - index: this.index, - body: { - mappings: { - properties: { - id: { type: 'keyword' }, - lastIndexedBlock: { type: 'long' } - } - } - } - }) + return await this.write(address, nonce) + } catch (error) { + const errorMsg = + `Error when creating new nonce entry ${nonce} for address ${address}: ` + + error.message + DATABASE_LOGGER.error(errorMsg) + return null + } + } + + // Documents are keyed by the checksummed address. Read every casing variant of the SAME + // address and keep the HIGHEST nonce: the nonce must stay monotonic, dropping back to 0 + // would briefly re-open the replay window that the nonce exists to close. + // A failed read throws: the callers must not take it for "no nonce yet" and start from 0. + async retrieve(address: string) { + const id = normalizeAddress(address) + try { + const client = await this.client() + let highest: number | null = null + for (const variant of addressCasingVariants(address)) { + const row = await getOrNull<{ nonce: number }>(client, this.index, variant) + const nonce = row?.source?.nonce + if (typeof nonce === 'number' && (highest === null || nonce > highest)) { + highest = nonce + } } - } catch (e) { - DATABASE_LOGGER.error(`Failed to create ES index '${this.index}': ${e.message}`) + return { id, nonce: highest } + } catch (error) { + const errorMsg = + `Error when retrieving nonce entry for address ${address}: ` + error.message + DATABASE_LOGGER.error(errorMsg) + throw error + } + } + + // A concurrent update of the same document fails with a version conflict, which also + // returns null. + async update(address: string, nonce: number) { + try { + const id = normalizeAddress(address) + const client = await this.client() + const { result } = await client.update({ + index: this.index, + id, + script: { + source: + 'if (ctx._source.nonce != null && ctx._source.nonce >= params.nonce) ' + + "{ ctx.op = 'noop' } else { ctx._source.nonce = params.nonce }", + params: { nonce } + }, + upsert: { nonce } + }) + return result === 'noop' ? null : { id, nonce } + } catch (error) { + const errorMsg = + `Error when updating nonce entry ${nonce} for address ${address}: ` + + error.message + DATABASE_LOGGER.error(errorMsg) + return null + } + } + + async delete(address: string) { + const id = normalizeAddress(address) + try { + const client = await this.client() + const row = await getOrNull<{ nonce: number }>(client, this.index, id) + if (!row) return { id, nonce: null } + await client.delete({ index: this.index, id }) + return { id, nonce: row.source.nonce } + } catch (error) { + const errorMsg = + `Error when deleting nonce entry for address ${address}: ` + error.message + DATABASE_LOGGER.error(errorMsg) + return null } } +} + +export class ElasticsearchIndexerDatabase extends AbstractIndexerDatabase { + private readonly index = elasticSchemas.indexerSchemas.index + + private constructor(config: OceanNodeDBConfig) { + super(config) + } + + static async create(config: OceanNodeDBConfig): Promise { + await ensureIndex(config, elasticSchemas.indexerSchemas) + return new ElasticsearchIndexerDatabase(config) + } + + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) + } async create(network: number, lastIndexedBlock: number) { try { - await this.client.index({ + const client = await this.client() + await client.index({ index: this.index, id: network.toString(), body: { lastIndexedBlock }, @@ -78,7 +189,8 @@ export class ElasticsearchIndexerDatabase extends AbstractIndexerDatabase { async retrieve(network: number) { try { - const result = await this.client.get({ + const client = await this.client() + const result = await client.get({ index: this.index, id: network.toString(), refresh: true @@ -93,13 +205,14 @@ export class ElasticsearchIndexerDatabase extends AbstractIndexerDatabase { async update(network: number, lastIndexedBlock: number) { try { - const exists = await this.client.exists({ + const client = await this.client() + const exists = await client.exists({ index: this.index, id: network.toString() }) if (exists) { - await this.client.update({ + await client.update({ index: this.index, id: network.toString(), body: { @@ -121,7 +234,8 @@ export class ElasticsearchIndexerDatabase extends AbstractIndexerDatabase { async delete(network: number) { try { - await this.client.delete({ + const client = await this.client() + await client.delete({ index: this.index, id: network.toString(), refresh: 'wait_for' @@ -136,44 +250,19 @@ export class ElasticsearchIndexerDatabase extends AbstractIndexerDatabase { } export class ElasticsearchDdoStateDatabase extends AbstractDdoStateDatabase { - private client: Client - private index: string + private readonly index = elasticSchemas.ddoStateSchema.index - constructor(config: OceanNodeDBConfig) { + private constructor(config: OceanNodeDBConfig) { super(config) - this.index = 'ddo_state' + } - return (async (): Promise => { - this.client = await createElasticsearchClientWithRetry(config) - await this.initializeIndex() - return this - })() as unknown as ElasticsearchDdoStateDatabase + static async create(config: OceanNodeDBConfig): Promise { + await ensureIndex(config, elasticSchemas.ddoStateSchema) + return new ElasticsearchDdoStateDatabase(config) } - private async initializeIndex() { - try { - const indexExists = await this.client.indices.exists({ index: this.index }) - if (!indexExists) { - await this.client.indices.create({ - index: this.index, - body: { - mappings: { - properties: { - id: { type: 'keyword' }, - chainId: { type: 'integer' }, - did: { type: 'keyword' }, - nft: { type: 'keyword' }, - txId: { type: 'keyword' }, - valid: { type: 'boolean' }, - error: { type: 'text' } - } - } - } - }) - } - } catch (e) { - DATABASE_LOGGER.error(`Failed to create ES index '${this.index}': ${e.message}`) - } + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) } async create( @@ -185,7 +274,8 @@ export class ElasticsearchDdoStateDatabase extends AbstractDdoStateDatabase { errorMsg: string = ' ' ) { try { - await this.client.index({ + const client = await this.client() + await client.index({ index: this.index, id: did, body: { chainId, did, nft: nftAddress, txId, valid, error: errorMsg }, @@ -202,7 +292,8 @@ export class ElasticsearchDdoStateDatabase extends AbstractDdoStateDatabase { async retrieve(did: string) { try { - const result = await this.client.get({ + const client = await this.client() + const result = await client.get({ index: this.index, id: did }) @@ -214,20 +305,19 @@ export class ElasticsearchDdoStateDatabase extends AbstractDdoStateDatabase { } } - async search(query: Record) { + // ten states at most, as the SQL store gives + async find(key: DdoStateKey) { try { - const result = await this.client.search({ + const client = await this.client() + const result = await client.search({ index: this.index, - body: { - query - } - }) - return result.hits.hits.map((hit: any) => { - return normalizeDocumentId(hit._source, hit._id) + query: { term: key as Record }, + size: 10 }) + return result.hits.hits.map((hit: any) => normalizeDocumentId(hit._source, hit._id)) } catch (error) { DATABASE_LOGGER.error( - `Error when searching ddo state (query ${safeStringify(query, { maxLength: 200 })}): ${error.message}` + `Error when searching ddo state (${safeStringify(key)}): ${error.message}` ) return null } @@ -242,13 +332,14 @@ export class ElasticsearchDdoStateDatabase extends AbstractDdoStateDatabase { errorMsg: string = ' ' ) { try { - const exists = await this.client.exists({ + const client = await this.client() + const exists = await client.exists({ index: this.index, id: did }) if (exists) { - await this.client.update({ + await client.update({ index: this.index, id: did, body: { @@ -271,7 +362,8 @@ export class ElasticsearchDdoStateDatabase extends AbstractDdoStateDatabase { async delete(did: string) { try { - await this.client.delete({ + const client = await this.client() + await client.delete({ index: this.index, id: did, // refresh: 'wait_for' @@ -287,47 +379,39 @@ export class ElasticsearchDdoStateDatabase extends AbstractDdoStateDatabase { } export class ElasticsearchOrderDatabase extends AbstractOrderDatabase { - private provider: Client + private readonly index = elasticSchemas.orderSchema.index - constructor(config: OceanNodeDBConfig, schema: ElasticsearchSchema) { - super(config, schema) - - return (async (): Promise => { - this.provider = await createElasticsearchClientWithRetry(config) - return this - })() as unknown as ElasticsearchOrderDatabase + private constructor(config: OceanNodeDBConfig) { + super(config) } - getSchema(): ElasticsearchSchema { - return this.schema as ElasticsearchSchema + static async create(config: OceanNodeDBConfig): Promise { + // without it, the first order would create the index with dynamic mappings + await ensureIndex(config, elasticSchemas.orderSchema) + return new ElasticsearchOrderDatabase(config) } - async search( - query: Record, - maxResultsPerPage?: number, - pageNumber?: number - ) { - try { - const { q, ...queryObj } = query + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) + } - const searchParams = { - index: this.getSchema().index, - body: { - query: { - match: q ? { _all: q } : queryObj - }, - from: (pageNumber - 1) * maxResultsPerPage || 0, - size: maxResultsPerPage || 10 - } - } - const result = await this.provider.search(searchParams) - return result.hits.hits.map((hit: any) => { - return normalizeDocumentId(hit._source, hit._id) - }) - } catch (error) { - const errorMsg = `Error when searching order entries: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null + // the newest orders first + async find(owner: OrderOwner, page: Page): Promise { + const [field, value] = Object.entries(owner)[0] + const client = await this.client() + const result = await client.search({ + index: this.index, + query: esQuery([{ paths: [field], anyOf: [value] }]), + sort: [{ timestamp: { order: 'desc' } }], + track_total_hits: true, + from: page.offset, + size: page.limit + }) + return { + results: result.hits.hits.map((hit: any) => + normalizeDocumentId(hit._source, hit._id) + ), + totalResults: totalOf(result.hits.total) } } @@ -343,6 +427,7 @@ export class ElasticsearchOrderDatabase extends AbstractOrderDatabase { startOrderId?: string ) { try { + const client = await this.client() const document = { orderId, type, @@ -354,8 +439,8 @@ export class ElasticsearchOrderDatabase extends AbstractOrderDatabase { did, startOrderId } - await this.provider.index({ - index: this.getSchema().index, + await client.index({ + index: this.index, id: orderId, body: document }) @@ -371,8 +456,9 @@ export class ElasticsearchOrderDatabase extends AbstractOrderDatabase { async retrieve(orderId: string) { try { - const result = await this.provider.get({ - index: this.getSchema().index, + const client = await this.client() + const result = await client.get({ + index: this.index, id: orderId }) return normalizeDocumentId(result._source, result._id) @@ -389,10 +475,11 @@ export class ElasticsearchOrderDatabase extends AbstractOrderDatabase { timestamp: number, consumer: string, payer: string, - datatokenAddress?: string, - startOrderId?: string + startOrderId?: string, + datatokenAddress?: string ) { try { + const client = await this.client() const document = { type, timestamp, @@ -401,8 +488,8 @@ export class ElasticsearchOrderDatabase extends AbstractOrderDatabase { datatokenAddress, startOrderId } - await this.provider.update({ - index: this.getSchema().index, + await client.update({ + index: this.index, id: orderId, body: { doc: document, @@ -421,8 +508,9 @@ export class ElasticsearchOrderDatabase extends AbstractOrderDatabase { async delete(orderId: string) { try { - await this.provider.delete({ - index: this.getSchema().index, + const client = await this.client() + await client.delete({ + index: this.index, id: orderId }) return { id: orderId } @@ -435,41 +523,26 @@ export class ElasticsearchOrderDatabase extends AbstractOrderDatabase { } export class ElasticsearchEscrowDatabase extends AbstractEscrowDatabase { - private provider: Client - - constructor(config: OceanNodeDBConfig, schema: ElasticsearchSchema) { - super(config, schema) + private readonly index = elasticSchemas.escrowSchema.index - return (async (): Promise => { - this.provider = await createElasticsearchClientWithRetry(config) - await this.initializeIndex() - return this - })() as unknown as ElasticsearchEscrowDatabase + private constructor(config: OceanNodeDBConfig) { + super(config) } - getSchema(): ElasticsearchSchema { - return this.schema as ElasticsearchSchema + static async create(config: OceanNodeDBConfig): Promise { + await ensureIndex(config, elasticSchemas.escrowSchema) + return new ElasticsearchEscrowDatabase(config) } - private async initializeIndex() { - try { - const { index } = this.getSchema() - const exists = await this.provider.indices.exists({ index }) - if (!exists) { - await this.provider.indices.create({ - index, - body: this.getSchema().body as any - }) - } - } catch (e) { - DATABASE_LOGGER.error(`Failed to create escrow index: ${e.message}`) - } + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) } async create(event: EscrowEvent) { try { - await this.provider.index({ - index: this.getSchema().index, + const client = await this.client() + await client.index({ + index: this.index, id: event.id, body: event }) @@ -483,8 +556,9 @@ export class ElasticsearchEscrowDatabase extends AbstractEscrowDatabase { async retrieve(id: string) { try { - const result = await this.provider.get({ - index: this.getSchema().index, + const client = await this.client() + const result = await client.get({ + index: this.index, id }) return normalizeDocumentId(result._source, result._id) @@ -497,18 +571,17 @@ export class ElasticsearchEscrowDatabase extends AbstractEscrowDatabase { async search(filters: Record, offset?: number, size?: number) { try { - // clamp the page size so a single request can't return an unbounded set - const limit = Math.min(size && size > 0 ? size : 100, 250) - const from = offset && offset > 0 ? offset : 0 + const client = await this.client() + const { from, limit } = escrowPage(offset, size) // Values may be equality scalars or RangeFilter objects (range on numeric fields). const query = buildElasticQuery(filters) const searchParams = { - index: this.getSchema().index, + index: this.index, body: { query, from, size: limit } } - const result = await this.provider.search(searchParams) + const result = await client.search(searchParams) return result.hits.hits.map((hit: any) => normalizeDocumentId(hit._source, hit._id)) } catch (error) { const errorMsg = @@ -521,8 +594,9 @@ export class ElasticsearchEscrowDatabase extends AbstractEscrowDatabase { async delete(id: string) { try { - await this.provider.delete({ - index: this.getSchema().index, + const client = await this.client() + await client.delete({ + index: this.index, id }) return { id } @@ -535,48 +609,30 @@ export class ElasticsearchEscrowDatabase extends AbstractEscrowDatabase { } export class ElasticsearchDdoDatabase extends AbstractDdoDatabase { - private client: Client - - constructor(config: OceanNodeDBConfig, schemas: ElasticsearchSchema[]) { - super(config, schemas) + private readonly schemas = elasticSchemas.ddoSchemas - return (async (): Promise => { - this.client = await createElasticsearchClientWithRetry(config) - await this.initializeIndexes() - return this - })() as unknown as ElasticsearchDdoDatabase + private constructor(config: OceanNodeDBConfig) { + super(config) } - private async initializeIndexes() { - for (const schema of this.getSchemas()) { - try { - const exists = await this.client.indices.exists({ index: schema.index }) - if (!exists) { - await this.client.indices.create({ - index: schema.index, - body: schema.body as any - }) - } - } catch (e) { - DATABASE_LOGGER.error(`Failed to create DDO index ${schema.index}: ${e.message}`) - } + static async create(config: OceanNodeDBConfig): Promise { + // a failure stops the startup, which tries again + for (const schema of elasticSchemas.ddoSchemas) { + await ensureIndex(config, schema) } + return new ElasticsearchDdoDatabase(config) + } + + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) } getSchemas(): ElasticsearchSchema[] { - return this.schemas as ElasticsearchSchema[] + return this.schemas } getDDOSchema(ddo: Record) { - let schemaName: string | undefined - const ddoInstance = DDOManager.getDDOClass(ddo) - const ddoData = ddoInstance.getDDOData() - - if ('indexedMetadata' in ddoData && ddoData?.indexedMetadata?.nft.state !== 0) { - schemaName = 'op_ddo_short' - } else if (ddo.version) { - schemaName = `op_ddo_v${ddo.version}` - } + const schemaName = ddoSchemaName(ddo) const schema = this.getSchemas().find((s) => s.index === schemaName) DATABASE_LOGGER.debug(`Returning schema: ${schemaName}`) return schema @@ -592,12 +648,13 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase { } try { - await this.client.delete({ + const client = await this.client() + await client.delete({ index: schema.index, id }) } catch (error) { - if (error.statusCode !== 404) { + if (!isStatus(error, 404)) { DATABASE_LOGGER.error( `Error when deleting stale DDO entry ${id} from schema ${schema.index}: ${error.message}` ) @@ -606,58 +663,55 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase { } } - async search(query: Record): Promise { - const results = [] - const maxPerPage = query.size || 100 - const from = (query.from || 1) * maxPerPage - maxPerPage + // the DDO indices of all versions + private indices(): string[] { + return this.getSchemas().map((schema) => schema.index) + } - if (query.index) { - const { index, ...queryWithoutIndex } = query - try { - const response = await this.client.search({ - index, - body: { - ...queryWithoutIndex, - from, - size: maxPerPage - } - }) - if (response.hits?.hits.length > 0) { - const nomalizedResponse = response.hits.hits.map((hit: any) => { - return normalizeDocumentId(hit._source, hit._id) - }) - results.push(nomalizedResponse) - } - } catch (error) { - const schemaErrorMsg = `Error for schema ${query.index}: ${error.message}` - DATABASE_LOGGER.error(schemaErrorMsg) - } - } else { - for (const schema of this.getSchemas()) { - try { - const response = await this.client.search({ - index: schema.index, - body: { - ...query, - from, - size: maxPerPage - } - }) - if (response.hits?.hits.length > 0) { - const nomalizedResponse = response.hits.hits.map((hit: any) => { - return normalizeDocumentId(hit._source, hit._id) - }) - results.push(nomalizedResponse) - } - } catch (error) { - const schemaErrorMsg = `Error for schema ${schema.index}: ${error.message}` - DATABASE_LOGGER.error(schemaErrorMsg) - continue - } - } + // One search over all DDO indices, so the sort, the page and the total cover all + // versions. It reads DDO indices only. + async find(conditions: Condition[], sort: SortSpec[], page: Page): Promise { + const client = await this.client() + const result = await client.search({ + index: this.indices(), + ignore_unavailable: true, + query: esQuery(conditions), + sort: esSort(sort, 'id'), + track_total_hits: true, + from: page.offset, + size: page.limit + }) + return { + results: result.hits.hits.map((hit: any) => + normalizeDocumentId(hit._source, hit._id) + ), + totalResults: totalOf(result.hits.total) } + } - return results + // one terms aggregation for each place of the tags (v4 and v5), added together + async tags(conditions: Condition[], limit: number): Promise { + const client = await this.client() + const aggs = Object.fromEntries( + DDO_FIELDS.tags.map((field, i) => [`tags${i}`, { terms: { field, size: limit } }]) + ) + const result = await client.search({ + index: this.indices(), + ignore_unavailable: true, + query: esQuery(conditions), + size: 0, + aggs + }) + const counts = new Map() + for (const agg of Object.values(result.aggregations ?? {}) as any[]) { + for (const bucket of agg.buckets ?? []) { + counts.set(bucket.key, (counts.get(bucket.key) ?? 0) + bucket.doc_count) + } + } + return [...counts] + .map(([tag, count]) => ({ tag, count })) + .sort((a, b) => b.count - a.count || (a.tag < b.tag ? -1 : 1)) + .slice(0, limit) } async create(ddo: Record): Promise { @@ -666,17 +720,14 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase { throw new Error(`Schema for version ${ddo.version} not found`) } try { + const client = await this.client() // avoid issue with nft fields, due to schema if (ddo?.indexedMetadata?.nft) delete ddo.nft const validation = await validateDDO(ddo) if (validation === true) { - const response = await this.client.index({ - index: schema.index, - id: ddo.id, - body: ddo - }) - return response + await client.index({ index: schema.index, id: ddo.id, body: ddo }) + return ddo } else { throw new Error(`Validation of DDO with schema version ${ddo.version} failed`) } @@ -691,7 +742,8 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase { let ddo = null for (const schema of this.getSchemas()) { try { - const response = await this.client.get({ + const client = await this.client() + const response = await client.get({ index: schema.index, id }) @@ -700,7 +752,7 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase { break } } catch (error) { - if (error.statusCode !== 404) { + if (!isStatus(error, 404)) { DATABASE_LOGGER.error( `Error when retrieving DDO entry ${id} from schema ${schema.index}: ${error.message}` ) @@ -715,44 +767,30 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase { return ddo } - // This is called from indexer "createOrUpdateDDO" - // we add the "id" field to match the response API with typesense - // since here we have an "_id" + // returns the DDO, as the SQL store does: the indexer reads its fields async update(ddo: Record): Promise { const schema = this.getDDOSchema(ddo) if (!schema) { throw new Error(`Schema for version ${ddo.version} not found`) } try { + const client = await this.client() // avoid issue with nft fields, due to schema if (ddo?.indexedMetadata?.nft) delete ddo.nft const validation = await validateDDO(ddo) if (validation === true) { - const response: any = await this.client.index({ - index: schema.index, - id: ddo.id, - body: ddo - }) + await client.index({ index: schema.index, id: ddo.id, body: ddo }) await this.deleteDDOFromOtherSchemas(ddo.id, schema) - // make sure we do not have different responses 4 between DBs - // do the same thing on other methods - if (response._id === ddo.id) { - return normalizeDocumentId(response, response._id) - } - return response + return ddo } else { throw new Error( `Validation of DDO with schema version ${ddo.version} failed with errors` ) } } catch (error) { - if (error.statusCode === 404) { + if (isStatus(error, 404)) { await this.delete(ddo.id) - const response = await this.create(ddo) - if (response._id === ddo.id) { - return normalizeDocumentId(response, response._id) - } - return response + return this.create(ddo) } const errorMsg = `Error when updating DDO entry ${ddo.id}: ${error.message}` DATABASE_LOGGER.error(errorMsg) @@ -764,19 +802,18 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase { let isDeleted = false for (const schema of this.getSchemas()) { try { - const response = await this.client.delete({ + const client = await this.client() + const response = await client.delete({ index: schema.index, id }) isDeleted = response.result === 'deleted' if (isDeleted) { - DATABASE_LOGGER.debug( - `Response for deleting the ddo: ${response.result}, isDeleted: ${isDeleted}` - ) - return response + DATABASE_LOGGER.debug(`Deleted DDO ${id} from schema ${schema.index}`) + return { id } } } catch (error) { - if (error.statusCode !== 404) { + if (!isStatus(error, 404)) { DATABASE_LOGGER.error( `Error when deleting DDO entry ${id} from schema ${schema.index}: ${error.message}` ) @@ -791,81 +828,46 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase { } } - async deleteAllAssetsFromChain(chainId: number, batchSize?: number): Promise { + // throws, as the SQL store does: a reindex must not continue over DDOs that are still there + async deleteAllAssetsFromChain(chainId: number): Promise { + const client = await this.client() let numDeleted = 0 for (const schema of this.getSchemas()) { try { - // add batch size logic - const response = await this.client.deleteByQuery({ - index: schema.index, - body: { - query: { - match: { chainId } - } - } - }) - - DATABASE_LOGGER.debug( - `Number of deleted ddos on schema ${schema.index}: ${response.deleted}` + numDeleted += await deleteMatching( + client, + schema.index, + esQuery([{ paths: DDO_FIELDS.chainId, anyOf: [chainId] }]) ) - - numDeleted += response.deleted } catch (error) { - if (error.statusCode !== 404) { - DATABASE_LOGGER.error( - `Error when deleting DDOs from schema ${schema.index}: ${error.message}` - ) - } + if (!isStatus(error, 404)) throw error } } - return numDeleted } } export class ElasticsearchLogDatabase extends AbstractLogDatabase { - private client: Client - private index: string + private readonly index = elasticSchemas.logSchemas.index - constructor(config: OceanNodeDBConfig) { + private constructor(config: OceanNodeDBConfig) { super(config) - this.index = 'log' + } - return (async (): Promise => { - this.client = await createElasticsearchClientWithRetry(config) - await this.initializeIndex() - return this - })() as unknown as ElasticsearchLogDatabase + static async create(config: OceanNodeDBConfig): Promise { + await ensureIndex(config, elasticSchemas.logSchemas) + return new ElasticsearchLogDatabase(config) } - private async initializeIndex() { - try { - const indexExists = await this.client.indices.exists({ index: this.index }) - if (!indexExists) { - await this.client.indices.create({ - index: this.index, - body: { - mappings: { - properties: { - timestamp: { type: 'date' }, - level: { type: 'keyword' }, - moduleName: { type: 'keyword' }, - message: { type: 'text' }, - meta: { type: 'object', enabled: false } - } - } - } - }) - } - } catch (e) { - DATABASE_LOGGER.error(`Failed to create ES index '${this.index}': ${e.message}`) - } + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) } async insertLog(logEntry: Record) { try { + const client = await this.client() const timestamp = new Date().toISOString() - const result = await this.client.index({ + const result = await client.index({ index: this.index, body: { ...logEntry, timestamp }, // refresh: 'wait_for' @@ -896,7 +898,8 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase { async retrieveLog(id: string): Promise | null> { try { - const result = await this.client.get({ + const client = await this.client() + const result = await client.get({ index: this.index, id }) @@ -917,6 +920,7 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase { page?: number ): Promise[]> { try { + const client = await this.client() const filterConditions: any = { bool: { must: [{ range: { timestamp: { gte: startTime, lte: endTime } } }] @@ -942,15 +946,14 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase { const from = currentPage * pageSize const size = Math.min(pageSize, maxLogs - allLogs.length) - // Elasticsearch result window limit - if (from + size > 10000) { + if (from + size > MAX_RESULT_WINDOW) { DATABASE_LOGGER.debug( - `Reached Elasticsearch result window limit (10000). Returning ${allLogs.length} logs.` + `Reached the result window limit (${MAX_RESULT_WINDOW}). Returning ${allLogs.length} logs.` ) break } - const result = await this.client.search({ + const result = await client.search({ index: this.index, body: { query: filterConditions, @@ -982,14 +985,15 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase { page: number, size: number ): Promise[]> { + const client = await this.client() const from = page * size - if (from + size > 10000) { + if (from + size > MAX_RESULT_WINDOW) { DATABASE_LOGGER.debug( - `Result window is too large, from + size must be less than or equal to: [10000]. "from": ${from}, "size": ${size}` + `Result window is too large, from + size must be less than or equal to: [${MAX_RESULT_WINDOW}]. "from": ${from}, "size": ${size}` ) return [] } - const result = await this.client.search({ + const result = await client.search({ index: this.index, body: { query: filterConditions, @@ -1006,7 +1010,8 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase { throw new Error('Log ID is required for deletion.') } try { - await this.client.delete({ + const client = await this.client() + await client.delete({ index: this.index, id: logId, // refresh: 'wait_for' @@ -1020,10 +1025,7 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase { } async deleteOldLogs(): Promise { - const defaultLogRetention = '2592000000' // 30 days in milliseconds - const currentTime = new Date().getTime() - const xTime = parseInt(process.env.LOG_RETENTION_TIME || defaultLogRetention) - const deleteBeforeTime = new Date(currentTime - xTime) + const deleteBeforeTime = new Date(Date.now() - this.retentionMs()) try { const oldLogs = await this.retrieveMultipleLogs(new Date(0), deleteBeforeTime, 200) @@ -1043,7 +1045,8 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase { async getLogsCount(): Promise { try { - const res = await this.client.count({ + const client = await this.client() + const res = await client.count({ index: this.index }) return res && res.count ? res.count : 0 @@ -1055,57 +1058,25 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase { } export class ElasticsearchAccessListDatabase extends AbstractAccessListDatabase { - private client: Client - private index: string + private readonly index = elasticSchemas.accessListSchema.index - constructor(config: OceanNodeDBConfig) { + private constructor(config: OceanNodeDBConfig) { super(config) - this.index = 'access_list' + } - return (async (): Promise => { - this.client = await createElasticsearchClientWithRetry(config) - await this.initializeIndex() - return this - })() as unknown as ElasticsearchAccessListDatabase + static async create( + config: OceanNodeDBConfig + ): Promise { + await ensureIndex(config, elasticSchemas.accessListSchema) + return new ElasticsearchAccessListDatabase(config) } - private docId(chainId: number, contractAddress: string): string { - return `${chainId}-${contractAddress.toLowerCase()}` + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) } - private async initializeIndex() { - try { - const indexExists = await this.client.indices.exists({ index: this.index }) - if (!indexExists) { - await this.client.indices.create({ - index: this.index, - body: { - mappings: { - properties: { - chainId: { type: 'integer' }, - contractAddress: { type: 'keyword' }, - name: { type: 'keyword' }, - symbol: { type: 'keyword' }, - transferable: { type: 'boolean' }, - users: { - type: 'nested', - properties: { - wallet: { type: 'keyword' }, - tokenId: { type: 'long' }, - block: { type: 'long' }, - txId: { type: 'keyword' } - } - }, - deploymentBlock: { type: 'long' }, - deploymentTxId: { type: 'keyword' } - } - } - } - }) - } - } catch (e) { - DATABASE_LOGGER.error(`Failed to create ES index '${this.index}': ${e.message}`) - } + private docId(chainId: number, contractAddress: string): string { + return `${chainId}-${contractAddress.toLowerCase()}` } async create( @@ -1120,7 +1091,8 @@ export class ElasticsearchAccessListDatabase extends AbstractAccessListDatabase const id = this.docId(chainId, contractAddress) const lowerContract = contractAddress.toLowerCase() try { - await this.client.update({ + const client = await this.client() + await client.update({ index: this.index, id, body: { @@ -1159,14 +1131,15 @@ export class ElasticsearchAccessListDatabase extends AbstractAccessListDatabase async retrieve(chainId: number, contractAddress: string) { const id = this.docId(chainId, contractAddress) try { - const result = await this.client.get({ + const client = await this.client() + const result = await client.get({ index: this.index, id, refresh: true }) return result._source } catch (error) { - if (error?.meta?.statusCode === 404) { + if (isStatus(error, 404)) { return null } const errorMsg = `Error when retrieving access list ${id}: ${error.message}` @@ -1180,7 +1153,8 @@ export class ElasticsearchAccessListDatabase extends AbstractAccessListDatabase const lowerContract = contractAddress.toLowerCase() const normalized: AccessListUser = { ...user, wallet: user.wallet.toLowerCase() } try { - await this.client.update({ + const client = await this.client() + await client.update({ index: this.index, id, body: { @@ -1216,7 +1190,8 @@ export class ElasticsearchAccessListDatabase extends AbstractAccessListDatabase async removeUserByTokenId(chainId: number, contractAddress: string, tokenId: number) { const id = this.docId(chainId, contractAddress) try { - await this.client.update({ + const client = await this.client() + await client.update({ index: this.index, id, body: { @@ -1234,7 +1209,7 @@ export class ElasticsearchAccessListDatabase extends AbstractAccessListDatabase }) return { id } } catch (error) { - if (error?.meta?.statusCode === 404) { + if (isStatus(error, 404)) { DATABASE_LOGGER.error( `AddressRemoved on missing access list ${id} (tokenId=${tokenId}); ignoring.` ) @@ -1260,7 +1235,8 @@ export class ElasticsearchAccessListDatabase extends AbstractAccessListDatabase filters.push({ term: { chainId } }) } try { - const result = await this.client.search({ + const client = await this.client() + const result = await client.search({ index: this.index, size: 250, body: { @@ -1278,7 +1254,8 @@ export class ElasticsearchAccessListDatabase extends AbstractAccessListDatabase async delete(chainId: number, contractAddress: string) { const id = this.docId(chainId, contractAddress) try { - await this.client.delete({ + const client = await this.client() + await client.delete({ index: this.index, id, refresh: 'wait_for' @@ -1293,8 +1270,7 @@ export class ElasticsearchAccessListDatabase extends AbstractAccessListDatabase } /** - * Make DB agnostic APIs. The response should be similar, no matter what DB engine is used - * Normalizes the document responses to match same kind of typesense ones + * Normalizes the document responses: adds an `id` property from the Elasticsearch `_id` * @param dbResult response from DB * @param _id id of the element * @returns result object with id property @@ -1305,5 +1281,3 @@ export function normalizeDocumentId(dbResult: any, _id?: any): any { } return dbResult } - -// ES client errors carry the HTTP status as statusCode (and meta.statusCode) diff --git a/src/components/database/ElasticSearchDdoStateQuery.ts b/src/components/database/ElasticSearchDdoStateQuery.ts deleted file mode 100644 index 324d2f85d..000000000 --- a/src/components/database/ElasticSearchDdoStateQuery.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { IDdoStateQuery } from '../../@types/DDO/IDdoStateQuery.js' - -export class ElasticSearchDdoStateQuery implements IDdoStateQuery { - buildQuery(did?: string, nft?: string, txId?: string): Record { - let query: Record = {} - - if (did) { - query = { - term: { - did - } - } - } - - if (nft) { - query = { - term: { - nft - } - } - } - - if (txId) { - query = { - term: { - txId - } - } - } - - return query - } -} diff --git a/src/components/database/ElasticSearchMetadataQuery.ts b/src/components/database/ElasticSearchMetadataQuery.ts deleted file mode 100644 index 5cfc74f2b..000000000 --- a/src/components/database/ElasticSearchMetadataQuery.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { SearchQuery } from './../../@types/DDO/SearchQuery.js' -import { IMetadataQuery } from '../../@types/DDO/IMetadataQuery.js' - -export class ElasticSearchMetadataQuery implements IMetadataQuery { - buildQuery(searchQuery: SearchQuery): Record { - if (this.isElasticSearchQuery(searchQuery)) { - return searchQuery - } - const elasticsearchQuery: Record = { - from: searchQuery.start || 0, - size: searchQuery.num_hits || 10, - query: { - bool: { - filter: [], - must_not: [] - } - } - } - if (searchQuery.filter_by) { - const filters = searchQuery.filter_by.split(' && ') - filters.forEach((filter: string) => { - let field, value - if (filter.includes('!=')) { - ;[field, value] = filter.split(':!=') - elasticsearchQuery.query.bool.must_not.push({ - term: { [field]: value } - }) - } else if (filter.includes(':=[')) { - ;[field, value] = filter.split(':=[') - const values = value.replace(']', '').split(',') - elasticsearchQuery.query.bool.filter.push({ - terms: { [field]: values } - }) - } else { - ;[field, value] = filter.split(':=') - elasticsearchQuery.query.bool.filter.push({ - term: { [field]: value } - }) - } - }) - } - if (searchQuery.sort_by) { - const [sortField, sortOrder] = searchQuery.sort_by.split(':') - elasticsearchQuery.sort = [{ [sortField]: { order: sortOrder } }] - } - - return elasticsearchQuery - } - - private isElasticSearchQuery(query: any): boolean { - return query && query.query && query.query.bool !== undefined - } -} diff --git a/src/components/database/ElasticsearchAuthToken.ts b/src/components/database/ElasticsearchAuthToken.ts new file mode 100644 index 000000000..c2bfd7333 --- /dev/null +++ b/src/components/database/ElasticsearchAuthToken.ts @@ -0,0 +1,87 @@ +import { Client } from '@elastic/elasticsearch' +import { OceanNodeDBConfig } from '../../@types/OceanNode.js' +import { DATABASE_LOGGER } from '../../utils/logging/common.js' +import { AuthToken, AuthTokenDatabaseProvider } from './AuthTokenDatabase.js' +import { + createElasticsearchClientWithRetry, + deleteMatching, + ensureIndex, + getOrNull, + isStatus +} from './ElasticsearchConfigHelper.js' +import { stateSchemas } from './ElasticSchemas.js' + +const INDEX = stateSchemas.authTokens.index + +// one document per token: id = sha256 of the token, the token itself is never stored +export class ElasticsearchAuthToken implements AuthTokenDatabaseProvider { + private constructor(private config: OceanNodeDBConfig) {} + + // resolved per call: the health monitor replaces the client after a reconnect + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) + } + + static async create(config: OceanNodeDBConfig): Promise { + await ensureIndex(config, stateSchemas.authTokens) + return new ElasticsearchAuthToken(config) + } + + async createToken( + tokenId: string, + address: string, + createdAt: number, + validUntil: number | null, + chainId?: string | null + ): Promise { + const body: AuthToken = { + address, + createdAt, + validUntil, + isValid: true, + chainId: chainId ?? null + } + const client = await this.client() + await client.index({ index: INDEX, id: tokenId, body, refresh: 'wait_for' }) + } + + // null for an unknown, invalidated or expired token; an expired one is invalidated + async validateTokenEntry(tokenId: string): Promise { + const row = await getOrNull(await this.client(), INDEX, tokenId) + if (!row || !row.source.isValid) { + return null + } + const entry = row.source + if (entry.validUntil !== null && entry.validUntil < Date.now()) { + DATABASE_LOGGER.debug('Expired auth token invalidated') + await this.invalidateTokenEntry(tokenId) + return null + } + return entry + } + + async invalidateTokenEntry(tokenId: string): Promise { + try { + const client = await this.client() + await client.update({ + index: INDEX, + id: tokenId, + doc: { isValid: false }, + refresh: 'wait_for' + }) + } catch (error) { + // an unknown token has nothing to invalidate, like the SQL UPDATE it replaces + if (isStatus(error, 404)) return + throw error + } + } + + async deleteExpiredTokens(now: number): Promise { + return deleteMatching(await this.client(), INDEX, { + bool: { + should: [{ range: { validUntil: { lt: now } } }, { term: { isValid: false } }], + minimum_should_match: 1 + } + }) + } +} diff --git a/src/components/database/ElasticsearchBucketRegistry.ts b/src/components/database/ElasticsearchBucketRegistry.ts new file mode 100644 index 000000000..ee98ec6ca --- /dev/null +++ b/src/components/database/ElasticsearchBucketRegistry.ts @@ -0,0 +1,103 @@ +import { Client } from '@elastic/elasticsearch' +import { OceanNodeDBConfig } from '../../@types/OceanNode.js' +import { MAX_RESULT_WINDOW } from './BaseDatabase.js' +import { BucketRegistry, BucketRow } from './BucketRegistry.js' +import { + createElasticsearchClientWithRetry, + ensureIndex, + getOrNull, + isStatus +} from './ElasticsearchConfigHelper.js' +import { stateSchemas } from './ElasticSchemas.js' + +const INDEX = stateSchemas.buckets.index + +type BucketDoc = Omit + +function toRow(bucketId: string, doc: BucketDoc): BucketRow { + return { + bucketId, + owner: doc.owner, + accessListJson: doc.accessListJson, + createdAt: doc.createdAt, + label: doc.label ?? null + } +} + +// one document per bucket: id = bucketId +export class ElasticsearchBucketRegistry implements BucketRegistry { + private constructor(private config: OceanNodeDBConfig) {} + + // resolved per call: the health monitor replaces the client after a reconnect + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) + } + + static async create(config: OceanNodeDBConfig): Promise { + await ensureIndex(config, stateSchemas.buckets) + return new ElasticsearchBucketRegistry(config) + } + + async upsertBucket( + bucketId: string, + owner: string, + accessListJson: string, + createdAt: number, + label: string | null + ): Promise { + const upsert: BucketDoc = { owner, accessListJson, createdAt, label } + const client = await this.client() + await client.update({ + index: INDEX, + id: bucketId, + doc: { accessListJson }, + upsert, + refresh: 'wait_for' + }) + } + + async getBucket(bucketId: string): Promise { + const row = await getOrNull(await this.client(), INDEX, bucketId) + return row ? toRow(bucketId, row.source) : null + } + + async listBucketsByOwner(owner: string): Promise { + const client = await this.client() + const result = await client.search({ + index: INDEX, + query: { term: { owner } }, + sort: [{ createdAt: { order: 'asc' } }], + // an owner with more buckets sees the oldest MAX_RESULT_WINDOW + size: MAX_RESULT_WINDOW + }) + return result.hits.hits.map((hit) => toRow(hit._id, hit._source)) + } + + async deleteBucket(bucketId: string): Promise { + try { + const client = await this.client() + await client.delete({ index: INDEX, id: bucketId, refresh: 'wait_for' }) + return true + } catch (error) { + if (isStatus(error, 404)) return false + throw error + } + } + + async updateBucketLabel( + bucketId: string, + owner: string, + label: string | null + ): Promise { + const client = await this.client() + const row = await getOrNull(client, INDEX, bucketId) + if (!row || row.source.owner !== owner) return false + await client.update({ + index: INDEX, + id: bucketId, + doc: { label }, + refresh: 'wait_for' + }) + return true + } +} diff --git a/src/components/database/ElasticsearchCompute.ts b/src/components/database/ElasticsearchCompute.ts new file mode 100644 index 000000000..22cdfdad5 --- /dev/null +++ b/src/components/database/ElasticsearchCompute.ts @@ -0,0 +1,652 @@ +import { Client, estypes } from '@elastic/elasticsearch' +import { + C2DStatusNumber, + C2DStatusText, + type DBComputeJob, + type ContainerMetricsSnapshot +} from '../../@types/C2D/C2D.js' +import { + ServiceStatusNumber, + SERVICE_START_PENDING_STATUSES, + type ServiceJob, + type ServiceModelDownload, + type ServiceReadiness +} from '../../@types/C2D/ServiceOnDemand.js' +import { OceanNodeDBConfig } from '../../@types/OceanNode.js' +import { DATABASE_LOGGER } from '../../utils/logging/common.js' +import type { ComputeDatabaseProvider, ServiceJobExpected } from './C2DDatabase.js' +import { + ACTIVE_SERVICE_STATUSES, + EXPIRABLE_SERVICE_STATUSES, + agreementIdVariants, + holdsReservation, + jobToBody, + matchesExpected, + oldImageCutoff, + withJobIdHash +} from './computeJobs.js' +import { MAX_RESULT_WINDOW } from './BaseDatabase.js' +import { + createElasticsearchClientWithRetry, + ensureIndex, + getOrNull, + isStatus, + searchAllPages +} from './ElasticsearchConfigHelper.js' +import { stateSchemas } from './ElasticSchemas.js' + +const JOBS_INDEX = stateSchemas.c2dJobs.index +const SERVICE_INDEX = stateSchemas.serviceJobs.index +const IMAGES_INDEX = stateSchemas.dockerImages.index +const LOCKS_INDEX = stateSchemas.serviceLocks.index +// replaces the listed top-level fields, like the SQL UPDATE ... SET did (a partial `doc` +// update would deep-merge the old body into the new one) +const SET_FIELDS_SCRIPT = 'ctx._source.putAll(params.fields)' +// lifecycle writes retry when they collide with a guarded body patch +const UPDATE_RETRIES = 3 +// Job and service job writes refresh at once (`true`, not 'wait_for'). A search sees the +// write when it returns in both cases, but 'wait_for' holds the write until the next +// scheduled refresh, up to 1 s. The engine holds the service lock across these writes, so a +// client that read the new status by id got "operation in progress" during that time. The +// indices are small and the writes are few, so the extra refreshes cost little. +const JOB_WRITE_REFRESH = true +const NEWEST_CREATED_FIRST: estypes.SortCombinations[] = [ + { dateCreated: { order: 'desc' } } +] + +// the jobId is the document id +type JobDoc = { + owner: string + did: string + dateCreated: number | null + dateFinished: number | null + status: number + statusText: string + inputDID: string[] + algoDID: string + agreementId: string + expireTimestamp: number + environment: string + body: any +} + +// the serviceId is the document id; acquiredAt in Unix ms +type LockDoc = { + holder: string + acquiredAt: number +} + +type ServiceJobDoc = { + owner: string + clusterHash: string + status: number + expiresAt: number + dateCreated: string + outputArchiveCount: number + body: ServiceJob +} + +// dates are stored as numbers so they compare and sort numerically, and read back as the +// decimal-seconds strings the callers use +function secondsToNumber(seconds: string | null | undefined): number | null { + return seconds ? Number(seconds) : null +} + +function numberToSeconds(seconds: number | null | undefined): string | null { + return seconds === null || seconds === undefined ? null : String(seconds) +} + +function docToJob(jobId: string, doc: JobDoc): DBComputeJob { + const { body, expireTimestamp, dateCreated, dateFinished, ...columns } = doc + return withJobIdHash({ + ...columns, + jobId, + dateCreated: numberToSeconds(dateCreated), + dateFinished: numberToSeconds(dateFinished), + ...body, + maxJobDuration: expireTimestamp + }) +} + +function serviceJobToDoc(job: ServiceJob): ServiceJobDoc { + return { + owner: job.owner, + clusterHash: job.clusterHash, + status: job.status, + expiresAt: job.expiresAt, + dateCreated: job.dateCreated, + outputArchiveCount: (job.outputArchives ?? []).length, + body: { ...job, updatedAt: Date.now() } + } +} + +function jobFilter( + agreementId?: string, + owner?: string +): estypes.QueryDslQueryContainer[] { + const filter: estypes.QueryDslQueryContainer[] = [] + if (agreementId) + filter.push({ terms: { agreementId: agreementIdVariants(agreementId) } }) + if (owner) filter.push({ term: { owner: owner.toLowerCase() } }) + return filter +} + +// the owner columns match case-insensitively (lowercase normalizer); same rule for reads by id +function sameAddress(stored: string | undefined, given: string): boolean { + return stored?.toLowerCase() === given.toLowerCase() +} + +function clusterFilter(clusterHash?: string): estypes.QueryDslQueryContainer[] { + return clusterHash ? [{ term: { clusterHash } }] : [] +} + +// jobs: id = jobId; service jobs and locks: id = serviceId; images: id = image name +export class ElasticsearchCompute implements ComputeDatabaseProvider { + private constructor(private config: OceanNodeDBConfig) {} + + // resolved per call: the health monitor replaces the client after a reconnect + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) + } + + static async create(config: OceanNodeDBConfig): Promise { + await ensureIndex(config, stateSchemas.c2dJobs) + await ensureIndex(config, stateSchemas.serviceJobs) + await ensureIndex(config, stateSchemas.dockerImages) + await ensureIndex(config, stateSchemas.serviceLocks) + return new ElasticsearchCompute(config) + } + + // ── C2D jobs ──────────────────────────────────────────────────────── + + async newJob(job: DBComputeJob): Promise { + const document: JobDoc = { + owner: job.owner, + did: job.did, + dateCreated: secondsToNumber(job.dateCreated || String(Date.now() / 1000)), + dateFinished: null, + status: job.status || C2DStatusNumber.JobStarted, + statusText: job.statusText || C2DStatusText.JobStarted, + inputDID: job.inputDID, + algoDID: job.algoDID, + agreementId: job.agreementId, + expireTimestamp: job.maxJobDuration, + environment: job.environment, + body: jobToBody(job) + } + const client = await this.client() + await client.create({ + index: JOBS_INDEX, + id: job.jobId, + document, + refresh: JOB_WRITE_REFRESH + }) + DATABASE_LOGGER.debug( + `C2D job ${job.clusterHash}-${job.jobId} created (env ${job.environment})` + ) + return job.jobId + } + + async getJob( + jobId?: string, + agreementId?: string, + owner?: string + ): Promise { + const jobs = jobId + ? await this.getJobById(jobId, agreementId, owner) + : await this.searchJobs({ bool: { filter: jobFilter(agreementId, owner) } }) + if (jobs.length === 0) { + DATABASE_LOGGER.debug( + `Could not find any job with jobId: ${jobId}, agreementId: ${agreementId}, or owner: ${owner} in database!` + ) + } + return jobs + } + + // real-time read by id; the other filters are applied to the document + private async getJobById( + jobId: string, + agreementId?: string, + owner?: string + ): Promise { + const row = await getOrNull(await this.client(), JOBS_INDEX, jobId) + if (!row) return [] + const doc = row.source + if (agreementId && !agreementIdVariants(agreementId).includes(doc.agreementId)) { + return [] + } + if (owner && !sameAddress(doc.owner, owner)) return [] + return [docToJob(jobId, doc)] + } + + // like the SQL UPDATE: only owner, status, statusText, maxJobDuration, body and + // dateFinished change; returns the number of updated jobs + async updateJob(job: DBComputeJob): Promise { + const fields: Partial = { + owner: job.owner ?? null, + status: job.status ?? null, + statusText: job.statusText ?? null, + expireTimestamp: job.maxJobDuration ?? null, + body: jobToBody(job), + dateFinished: secondsToNumber(job.dateFinished) + } + return await this.setFields(JOBS_INDEX, job.jobId, fields, `job ${job.jobId}`) + } + + async getRunningJobs(engine?: string, environment?: string): Promise { + const jobs = await this.searchJobs( + { bool: { must_not: [{ exists: { field: 'dateFinished' } }] } }, + [{ dateCreated: { order: 'asc' } }], + true + ) + return jobs.filter( + (job) => + (!engine || engine === job.clusterHash) && + (!environment || environment === job.environment) + ) + } + + async deleteJob(jobId: string): Promise { + try { + const client = await this.client() + await client.delete({ index: JOBS_INDEX, id: jobId, refresh: JOB_WRITE_REFRESH }) + return true + } catch (error) { + if (isStatus(error, 404)) return false + throw error + } + } + + async getFinishedJobs(environments?: string[]): Promise { + const filter: estypes.QueryDslQueryContainer[] = [ + { exists: { field: 'dateFinished' } }, + ...environmentFilter(environments) + ] + return await this.searchJobs( + { bool: { filter } }, + [{ dateFinished: { order: 'desc' } }], + true + ) + } + + /** @param fromTimestamp lower bound in Unix seconds — see C2DDatabase.getJobs. */ + async getJobs( + environments?: string[], + fromTimestamp?: number, + consumerAddrs?: string[], + status?: C2DStatusNumber, + runningJobs?: boolean + ): Promise { + const filter = environmentFilter(environments) + const hasFrom = fromTimestamp !== undefined && fromTimestamp !== null + if (runningJobs) { + filter.push({ term: { status: C2DStatusNumber.RunningAlgorithm } }) + if (hasFrom) filter.push({ range: { dateCreated: { gte: fromTimestamp } } }) + } else { + if (hasFrom) filter.push({ range: { dateFinished: { gte: fromTimestamp } } }) + // C2DStatusNumber.JobStarted is 0, so a truthiness check would silently drop the + // filter and return every status instead. + if (status !== undefined && status !== null) filter.push({ term: { status } }) + } + if (consumerAddrs && consumerAddrs.length > 0) { + filter.push({ + terms: { owner: consumerAddrs.map((address) => address.toLowerCase()) } + }) + } + return await this.searchJobs({ bool: { filter } }, NEWEST_CREATED_FIRST) + } + + async getJobsByStatus( + environments: string[], + status: C2DStatusNumber[] + ): Promise { + const filter = environmentFilter(environments) + if (status && status.length > 0) filter.push({ terms: { status } }) + return await this.searchJobs({ bool: { filter } }, NEWEST_CREATED_FIRST, true) + } + + private async searchJobs( + query: estypes.QueryDslQueryContainer, + sort?: estypes.SortCombinations[], + allPages: boolean = false + ): Promise { + const request = { index: JOBS_INDEX, query, sort } + const hits = await this.searchHits(request, allPages) + return hits.map((hit) => docToJob(hit._id, hit._source)) + } + + // Returns one page of MAX_RESULT_WINDOW hits. `allPages` reads every match: the engine + // loops and the cleanup need it, the public job lists keep the one-page limit. + private async searchHits( + request: Parameters[1], + allPages: boolean + ): Promise[]> { + const client = await this.client() + if (allPages) return await searchAllPages(client, request) + return (await client.search({ ...request, size: MAX_RESULT_WINDOW })).hits.hits + } + + // Replaces the listed top-level fields, like the SQL UPDATE; returns the number of updated + // documents. The script replaces the fields fully, so a retry keeps last-write-wins against + // a concurrent guarded body patch. + private async setFields( + index: string, + id: string, + fields: Record, + what: string + ): Promise { + try { + const client = await this.client() + await client.update({ + index, + id, + script: { source: SET_FIELDS_SCRIPT, params: { fields } }, + retry_on_conflict: UPDATE_RETRIES, + refresh: JOB_WRITE_REFRESH + }) + return 1 + } catch (error) { + if (isStatus(error, 404)) return 0 + DATABASE_LOGGER.error(`Error while updating ${what}: ${error.message}`) + throw error + } + } + + // ── Docker images ─────────────────────────────────────────────────── + + async updateImage(image: string): Promise { + const client = await this.client() + await client.index({ + index: IMAGES_INDEX, + id: image, + document: { lastUsedTimestamp: Math.floor(Date.now() / 1000) }, + refresh: 'wait_for' + }) + DATABASE_LOGGER.debug(`Updated image usage timestamp for ${image}`) + } + + async deleteImage(image: string): Promise { + try { + const client = await this.client() + await client.delete({ index: IMAGES_INDEX, id: image, refresh: 'wait_for' }) + DATABASE_LOGGER.debug(`Deleted image ${image}`) + } catch (error) { + if (!isStatus(error, 404)) throw error + } + } + + async getOldImages(retentionDays: number): Promise { + const cutoff = oldImageCutoff(retentionDays) + // every page: images that Docker cannot remove stay in the index and must not hide + // the later ones + const hits = await searchAllPages(await this.client(), { + index: IMAGES_INDEX, + query: { range: { lastUsedTimestamp: { lt: cutoff } } }, + sort: [{ lastUsedTimestamp: { order: 'asc' } }], + _source: false + }) + return hits.map((hit) => hit._id) + } + + // ── Service-on-Demand jobs ────────────────────────────────────────── + + async newServiceJob(job: ServiceJob): Promise { + const client = await this.client() + await client.create({ + index: SERVICE_INDEX, + id: job.serviceId, + document: serviceJobToDoc(job), + refresh: JOB_WRITE_REFRESH + }) + } + + async getServiceJob(serviceId?: string, owner?: string): Promise { + // callers are checksum-normalized on ingress, but older rows may hold another casing + if (!serviceId) { + return await this.searchServiceJobs( + owner ? [{ term: { owner: owner.toLowerCase() } }] : [] + ) + } + const row = await getOrNull( + await this.client(), + SERVICE_INDEX, + serviceId + ) + if (!row || (owner && !sameAddress(row.source.owner, owner))) return [] + return [row.source.body] + } + + // like the SQL UPDATE: every column but dateCreated; returns the number of updated jobs + async updateServiceJob(job: ServiceJob): Promise { + const { dateCreated, ...fields } = serviceJobToDoc(job) + return await this.setFields( + SERVICE_INDEX, + job.serviceId, + fields, + `service job ${job.serviceId}` + ) + } + + async updateServiceJobMetrics( + serviceId: string, + expected: ServiceJobExpected, + runtimeMetrics: ContainerMetricsSnapshot + ): Promise { + return await this.patchServiceJobBody(serviceId, expected, 'metrics', (body) => { + body.runtimeMetrics = runtimeMetrics + }) + } + + // Same guarantees as updateServiceJobMetrics, for the readiness probe result: it is sampled + // from the same lease-free background loop, so it must never overwrite a lifecycle + // transition either. `readiness` is undefined for a service the node cannot probe. + async updateServiceJobReadiness( + serviceId: string, + expected: ServiceJobExpected, + readiness: ServiceReadiness | undefined, + modelDownload?: ServiceModelDownload + ): Promise { + return await this.patchServiceJobBody(serviceId, expected, 'readiness', (body) => { + if (readiness) body.readiness = readiness + // Only overwritten when a fresh sample was taken: once the engine is ready the walk + // stops, and the last figures stay as the record of what was downloaded. + if (modelDownload) body.modelDownload = modelDownload + }) + } + + // Persists ONE best-effort, non-lifecycle field onto a service job without clobbering + // lifecycle fields another process may have changed: reads the current document, checks + // owner/clusterHash/status/containerId, applies `patch` to the current body and writes it + // back only if the document is unchanged since the read (if_seq_no/if_primary_term). A + // lifecycle write in between makes the write fail with 409. Returns true when written. + private async patchServiceJobBody( + serviceId: string, + expected: ServiceJobExpected, + what: string, + patch: (body: ServiceJob) => void + ): Promise { + try { + const client = await this.client() + const current = await getOrNull(client, SERVICE_INDEX, serviceId) + if (!current) return false + const { body } = current.source + if (!matchesExpected(body, expected)) return false + patch(body) + await client.index({ + index: SERVICE_INDEX, + id: serviceId, + document: { + ...current.source, + outputArchiveCount: (body.outputArchives ?? []).length, + body: { ...body, updatedAt: Date.now() } + }, + if_seq_no: current.seqNo, + if_primary_term: current.primaryTerm, + refresh: JOB_WRITE_REFRESH + }) + return true + } catch (error) { + if (isStatus(error, 409)) return false + DATABASE_LOGGER.error( + `Error while updating ${what} of service job ${serviceId}: ${error.message}` + ) + return false + } + } + + async getRunningServiceJobs(clusterHash?: string): Promise { + const jobs = await this.searchServiceJobs( + [{ terms: { status: ACTIVE_SERVICE_STATUSES } }, ...clusterFilter(clusterHash)], + true + ) + // filtered here: the payment lives in the body + return jobs.filter(holdsReservation) + } + + async getExpiredServiceJobs(clusterHash?: string): Promise { + return await this.searchServiceJobs( + [ + { terms: { status: EXPIRABLE_SERVICE_STATUSES } }, + { range: { expiresAt: { lte: Date.now() } } }, + ...clusterFilter(clusterHash) + ], + true + ) + } + + // Expired (terminal) services whose paid window ended at or before `expiresBefore` (Unix + // ms) and that still hold output archives — candidates for deleting them once storage + // expiry has elapsed. Expired jobs are kept forever (with outputArchives: [] once + // cleaned), so the archive count is filtered in the query. + async getExpiredServiceJobsBefore( + expiresBefore: number, + clusterHash?: string + ): Promise { + return await this.searchServiceJobs( + [ + { term: { status: ServiceStatusNumber.Expired } }, + { range: { expiresAt: { lte: expiresBefore } } }, + { range: { outputArchiveCount: { gt: 0 } } }, + ...clusterFilter(clusterHash) + ], + true + ) + } + + // Service jobs that are mid-start and need the background loop to advance them, including + // the intermediate states so the loop can resume them after a node restart. + async getPendingServiceStarts(clusterHash?: string): Promise { + return await this.searchServiceJobs( + [ + { terms: { status: [...SERVICE_START_PENDING_STATUSES] } }, + ...clusterFilter(clusterHash) + ], + true + ) + } + + private async searchServiceJobs( + filter: estypes.QueryDslQueryContainer[], + allPages: boolean = false + ): Promise { + const request = { index: SERVICE_INDEX, query: { bool: { filter } } } + const hits = await this.searchHits(request, allPages) + return hits.map((hit) => hit._source.body) + } + + // ── Service lifecycle locks ───────────────────────────────────────── + + // Takes the lock for serviceId: creates the lock document, or takes over a lock whose + // acquiredAt is at least staleMs old (crashed holder). The takeover writes only if the + // document is unchanged since the read (if_seq_no/if_primary_term), so two instances + // racing for one stale lock can never both win. The lock now works across hosts that + // share the Elasticsearch cluster; claims of queued C2D jobs stay in issue #1553. + // A lock released between the create and the read returns false: the callers retry. + async acquireServiceLock( + serviceId: string, + holder: string, + staleMs: number + ): Promise { + // refresh: true, not wait_for: refreshServiceLocks searches, so it must see the new + // lock; lock writes are rare and tiny, and this skips the up-to-1 s refresh wait + const write = { index: LOCKS_INDEX, id: serviceId, refresh: true } + const client = await this.client() + try { + await client.index({ + ...write, + document: { holder, acquiredAt: Date.now() }, + op_type: 'create' + }) + return true + } catch (error) { + if (!isStatus(error, 409)) throw error + } + const current = await getOrNull(client, LOCKS_INDEX, serviceId) + if (!current || current.source.acquiredAt > Date.now() - staleMs) return false + try { + await client.index({ + ...write, + document: { holder, acquiredAt: Date.now() }, + if_seq_no: current.seqNo, + if_primary_term: current.primaryTerm + }) + return true + } catch (error) { + if (isStatus(error, 409)) return false + throw error + } + } + + // Releases only a lock this holder still holds: a stale lock taken over by another + // instance must not be deleted out from under its new holder. A conflict can also come + // from our own heartbeat, which re-stamps the lock: read it again and retry while the + // holder is still ours. + async releaseServiceLock(serviceId: string, holder: string): Promise { + const client = await this.client() + for (;;) { + const current = await getOrNull(client, LOCKS_INDEX, serviceId) + if (!current || current.source.holder !== holder) return + try { + await client.delete({ + index: LOCKS_INDEX, + id: serviceId, + if_seq_no: current.seqNo, + if_primary_term: current.primaryTerm + }) + return + } catch (error) { + if (isStatus(error, 404)) return + if (!isStatus(error, 409)) throw error + } + } + } + + // Heartbeat: re-stamps every lock this holder owns, so long operations (multi-minute + // image pulls and builds) are not taken over as stale. + async refreshServiceLocks(holder: string): Promise { + const client = await this.client() + await client.updateByQuery({ + index: LOCKS_INDEX, + query: { term: { holder } }, + script: { + source: 'ctx._source.acquiredAt = params.now', + params: { now: Date.now() } + }, + refresh: true, + conflicts: 'proceed' + }) + } + + // True while any instance holds a fresh lock on serviceId; read-only observers (the + // container health check) use it to skip a service another instance is restarting. + async isServiceLocked(serviceId: string, staleMs: number): Promise { + const current = await getOrNull(await this.client(), LOCKS_INDEX, serviceId) + return !!current && current.source.acquiredAt > Date.now() - staleMs + } +} + +function environmentFilter(environments?: string[]): estypes.QueryDslQueryContainer[] { + return environments && environments.length > 0 + ? [{ terms: { environment: environments } }] + : [] +} diff --git a/src/components/database/ElasticsearchConfigDatabase.ts b/src/components/database/ElasticsearchConfigDatabase.ts new file mode 100644 index 000000000..53df94e23 --- /dev/null +++ b/src/components/database/ElasticsearchConfigDatabase.ts @@ -0,0 +1,53 @@ +import { Client } from '@elastic/elasticsearch' +import { OceanNodeDBConfig } from '../../@types/OceanNode.js' +import { DATABASE_LOGGER } from '../../utils/logging/common.js' +import { ConfigDatabase } from './ConfigDatabase.js' +import { + createElasticsearchClientWithRetry, + ensureIndex, + getOrNull +} from './ElasticsearchConfigHelper.js' +import { stateSchemas } from './ElasticSchemas.js' + +const INDEX = stateSchemas.nodeConfig.index + +// one document per key: id = key, body = { value } +export class ElasticsearchConfigDatabase implements ConfigDatabase { + private constructor(private config: OceanNodeDBConfig) {} + + // resolved per call: the health monitor replaces the client after a reconnect + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) + } + + static async create(config: OceanNodeDBConfig): Promise { + await ensureIndex(config, stateSchemas.nodeConfig) + return new ElasticsearchConfigDatabase(config) + } + + async createOrUpdateConfig(key: string, value: string) { + try { + const client = await this.client() + await client.index({ + index: INDEX, + id: key, + body: { value }, + refresh: 'wait_for' + }) + return { key, value } + } catch (error) { + DATABASE_LOGGER.error(`Error when writing config value ${key}: ${error.message}`) + return null + } + } + + async retrieveValue(key: string) { + try { + const row = await getOrNull<{ value: string }>(await this.client(), INDEX, key) + return { value: row?.source?.value ?? null } + } catch (error) { + DATABASE_LOGGER.error(`Error when reading config value ${key}: ${error.message}`) + return null + } + } +} diff --git a/src/components/database/ElasticsearchConfigHelper.ts b/src/components/database/ElasticsearchConfigHelper.ts index 0d3442e9f..be504284f 100644 --- a/src/components/database/ElasticsearchConfigHelper.ts +++ b/src/components/database/ElasticsearchConfigHelper.ts @@ -1,18 +1,14 @@ -import EventEmitter from 'node:events' -import { Client } from '@elastic/elasticsearch' +import { Client, estypes } from '@elastic/elasticsearch' import { OceanNodeDBConfig } from '../../@types' import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { LOG_LEVELS_STR } from '../../utils/logging/Logger.js' -import { DB_TYPES } from '../../utils/constants.js' +import { MAX_RESULT_WINDOW } from './BaseDatabase.js' +import { ElasticsearchSchema } from './ElasticSchemas.js' +import { DB_CONNECTION_EVENTS, DB_EVENTS } from './connectionEvents.js' -export const DB_EVENTS = { - CONNECTION_LOST: 'db:connection:lost', - CONNECTION_RESTORED: 'db:connection:restored' -} as const -export const ES_CONNECTION_EVENTS = new EventEmitter() const ES_CONNECT_FAILED_KEY = 'es-connect-failed' -export interface ElasticsearchRetryConfig { +interface ElasticsearchRetryConfig { requestTimeout?: number pingTimeout?: number resurrectStrategy?: 'ping' | 'optimistic' | 'none' @@ -23,7 +19,7 @@ export interface ElasticsearchRetryConfig { healthCheckInterval?: number } -export const DEFAULT_ELASTICSEARCH_CONFIG: Required = { +const DEFAULT_ELASTICSEARCH_CONFIG: Required = { requestTimeout: parseInt(process.env.ELASTICSEARCH_REQUEST_TIMEOUT || '30000'), pingTimeout: parseInt(process.env.ELASTICSEARCH_PING_TIMEOUT || '3000'), resurrectStrategy: @@ -46,9 +42,7 @@ class ElasticsearchClientSingleton { private client: Client | null = null private config: OceanNodeDBConfig | null = null private connectionAttempts: number = 0 - private lastConnectionTime: number = 0 - private isRetrying: boolean = false - private healthCheckTimer: NodeJS.Timeout | null = null + private isChecking: boolean = false private isMonitoring: boolean = false private connectionLostEmitted: boolean = false @@ -61,117 +55,98 @@ class ElasticsearchClientSingleton { return ElasticsearchClientSingleton.instance } - private isElasticsearchDatabase(config: OceanNodeDBConfig): boolean { - return config.dbType === DB_TYPES.ELASTIC_SEARCH - } - - public async getClient( - config: OceanNodeDBConfig, - customConfig: Partial = {} - ): Promise { - if (!this.isElasticsearchDatabase(config)) { - DATABASE_LOGGER.debug( - `Skipping Elasticsearch connection - database type is set to '${config.dbType || 'unknown'}', not '${DB_TYPES.ELASTIC_SEARCH}'` - ) - throw new Error(`Database type '${config.dbType}' is not Elasticsearch`) - } - + public async getClient(config: OceanNodeDBConfig): Promise { if (this.client && this.config) { - // Skip the extra ping here: 5 DB-class constructors all call getClient() - // during reconnect reinit, and concurrent pings cause false errors that trigger another LOST/RESTORED cycle. + // no ping here: every store call gets the client, and concurrent pings give false + // errors that start another LOST/RESTORED cycle if (this.isMonitoring) { return this.client } const isHealthy = await this.checkConnectionHealth() if (isHealthy) { - this.startHealthMonitoring(config, customConfig) + this.startHealthMonitoring(config) return this.client } else { DATABASE_LOGGER.warn( `Elasticsearch connection unhealthy at ${this.maskUrl(this.config.url)} - health monitoring will handle reconnection` ) - this.startHealthMonitoring(config, customConfig) + this.startHealthMonitoring(config) throw new Error('Elasticsearch connection is not healthy') } } - const client = await this.createNewConnection(config, customConfig) - this.startHealthMonitoring(config, customConfig) + // The health monitor is the only reconnect path while it runs: its reconnect resets + // connectionLostEmitted and emits CONNECTION_RESTORED, which restarts the indexers. A + // reconnect from a store call would skip both, so store calls fail fast here instead. + if (this.isMonitoring) { + throw new Error('Elasticsearch connection not available (reconnecting)') + } + + const client = await this.createNewConnection(config) + this.startHealthMonitoring(config) return client } - private startHealthMonitoring( - config: OceanNodeDBConfig, - customConfig: Partial = {} - ): void { - if (this.isMonitoring || !this.client || !this.isElasticsearchDatabase(config)) return - - const finalConfig = { - ...DEFAULT_ELASTICSEARCH_CONFIG, - ...customConfig - } + private startHealthMonitoring(config: OceanNodeDBConfig): void { + if (this.isMonitoring || !this.client) return + const { healthCheckInterval } = DEFAULT_ELASTICSEARCH_CONFIG this.isMonitoring = true DATABASE_LOGGER.debug( - `Starting Elasticsearch health monitoring (interval: ${finalConfig.healthCheckInterval}ms)` + `Starting Elasticsearch health monitoring (interval: ${healthCheckInterval}ms)` ) - this.healthCheckTimer = setInterval(async () => { - if (this.isRetrying) { - return + // One check at a time, ping included: a ping can take longer than the interval, and a + // check that started on the old client must not close the client of a reconnect. + setInterval(async () => { + if (this.isChecking) return + this.isChecking = true + try { + await this.checkAndReconnect(config) + } finally { + this.isChecking = false } + }, healthCheckInterval) + } - const isHealthy = await this.checkConnectionHealth() - if (!isHealthy) { - if (this.client) { - try { - this.client.close() - } catch (err) { - DATABASE_LOGGER.error( - `Error closing Elasticsearch client during health check: ${err instanceof Error ? err.message : String(err)}` - ) - } - this.client = null - this.config = null - } - - // Emit CONNECTION_LOST - if (!this.connectionLostEmitted) { - this.connectionLostEmitted = true - DATABASE_LOGGER.warn( - `Elasticsearch connection lost to ${this.maskUrl(config.url)} - starting reconnection attempts every ${finalConfig.healthCheckInterval}ms` - ) - ES_CONNECTION_EVENTS.emit(DB_EVENTS.CONNECTION_LOST) - } - - // Single reconnection attempt - this.isRetrying = true - try { - DATABASE_LOGGER.debug( - `Attempting Elasticsearch reconnection to ${this.maskUrl(config.url)}` - ) - await this.createNewConnection(config, customConfig) - this.isRetrying = false - this.connectionLostEmitted = false - DATABASE_LOGGER.info( - `Elasticsearch connection restored to ${this.maskUrl(config.url)}` - ) - ES_CONNECTION_EVENTS.emit(DB_EVENTS.CONNECTION_RESTORED) - } catch { - this.isRetrying = false - } + private async checkAndReconnect(config: OceanNodeDBConfig): Promise { + if (await this.checkConnectionHealth()) return + + if (this.client) { + try { + this.client.close() + } catch (err) { + DATABASE_LOGGER.error( + `Error closing Elasticsearch client during health check: ${err instanceof Error ? err.message : String(err)}` + ) } - }, finalConfig.healthCheckInterval) - } + this.client = null + this.config = null + } + + if (!this.connectionLostEmitted) { + this.connectionLostEmitted = true + DATABASE_LOGGER.warn( + `Elasticsearch connection lost to ${this.maskUrl(config.url)} - starting reconnection attempts every ${DEFAULT_ELASTICSEARCH_CONFIG.healthCheckInterval}ms` + ) + DB_CONNECTION_EVENTS.emit(DB_EVENTS.CONNECTION_LOST) + } - private stopHealthMonitoring(): void { - if (this.healthCheckTimer) { - clearInterval(this.healthCheckTimer) - this.healthCheckTimer = null - this.isMonitoring = false - DATABASE_LOGGER.debug(`Stopped Elasticsearch connection monitoring`) + // a single reconnection attempt; the next interval tries again + try { + DATABASE_LOGGER.debug( + `Attempting Elasticsearch reconnection to ${this.maskUrl(config.url)}` + ) + await this.createNewConnection(config) + } catch { + return } + this.connectionLostEmitted = false + DATABASE_LOGGER.info( + `Elasticsearch connection restored to ${this.maskUrl(config.url)}` + ) + DB_CONNECTION_EVENTS.emit(DB_EVENTS.CONNECTION_RESTORED) } private async checkConnectionHealth(): Promise { @@ -186,22 +161,9 @@ class ElasticsearchClientSingleton { } } - private async createNewConnection( - config: OceanNodeDBConfig, - customConfig: Partial = {} - ): Promise { - if (!this.isElasticsearchDatabase(config)) { - throw new Error(`Database type '${config.dbType}' is not Elasticsearch`) - } - + private async createNewConnection(config: OceanNodeDBConfig): Promise { this.connectionAttempts++ - this.lastConnectionTime = Date.now() - - const finalConfig = { - ...DEFAULT_ELASTICSEARCH_CONFIG, - ...customConfig - } - + const settings = DEFAULT_ELASTICSEARCH_CONFIG try { const client = new Client({ node: config.url, @@ -209,13 +171,13 @@ class ElasticsearchClientSingleton { config.username && config.password ? { username: config.username, password: config.password } : undefined, - requestTimeout: finalConfig.requestTimeout, - pingTimeout: finalConfig.pingTimeout, - resurrectStrategy: finalConfig.resurrectStrategy, - maxRetries: finalConfig.maxRetries, - sniffOnStart: finalConfig.sniffOnStart, - sniffInterval: finalConfig.sniffInterval, - sniffOnConnectionFault: finalConfig.sniffOnConnectionFault + requestTimeout: settings.requestTimeout, + pingTimeout: settings.pingTimeout, + resurrectStrategy: settings.resurrectStrategy, + maxRetries: settings.maxRetries, + sniffOnStart: settings.sniffOnStart, + sniffInterval: settings.sniffInterval, + sniffOnConnectionFault: settings.sniffOnConnectionFault }) await client.ping() @@ -248,62 +210,125 @@ class ElasticsearchClientSingleton { return url.replace(/\/\/[^@]+@/, '//***:***@') } } +} - private closeConnectionSync(): void { - this.stopHealthMonitoring() - if (this.client) { - try { - this.client.close() - } catch (error) { - // silent close, no logging needed - } - this.client = null - this.config = null - } - this.connectionLostEmitted = false - } +export function createElasticsearchClientWithRetry( + config: OceanNodeDBConfig +): Promise { + return ElasticsearchClientSingleton.getInstance().getClient(config) +} - public getConnectionStats(): { - attempts: number - lastConnection: number - connected: boolean - isRetrying: boolean - isMonitoring: boolean - } { - return { - attempts: this.connectionAttempts, - lastConnection: this.lastConnectionTime, - connected: this.client !== null, - isRetrying: this.isRetrying, - isMonitoring: this.isMonitoring - } +// true when the request failed with this HTTP status +export function isStatus(error: any, statusCode: number): boolean { + return error?.meta?.statusCode === statusCode +} + +// Deletes the documents that match and returns how many. A delete by query does not roll +// back: it can delete some documents and fail, time out or meet a changed document for the +// others, so it throws then (a version conflict throws from the client). +export async function deleteMatching( + client: Client, + index: string, + query: estypes.QueryDslQueryContainer +): Promise { + const result = await client.deleteByQuery({ index, query, refresh: true }) + if (result.timed_out) { + throw new Error( + `Delete on index ${index} timed out after deleting ${result.deleted ?? 0} documents` + ) } + if (result.failures?.length > 0) { + throw new Error( + `Delete on index ${index} failed for ${result.failures.length} documents after ` + + `deleting ${result.deleted ?? 0}: ${result.failures[0].cause?.reason}` + ) + } + return result.deleted ?? 0 } -export async function createElasticsearchClientWithRetry( +// creates the index with its mappings when it does not exist yet; throws on failure +export async function ensureIndex( config: OceanNodeDBConfig, - customConfig: Partial = {} -): Promise { - const singleton = ElasticsearchClientSingleton.getInstance() - return await singleton.getClient(config, customConfig) + schema: ElasticsearchSchema +): Promise { + const client = await createElasticsearchClientWithRetry(config) + if (await client.indices.exists({ index: schema.index })) return + try { + await client.indices.create({ index: schema.index, body: schema.body as any }) + } catch (error) { + // another instance created it between the two calls + if (error?.meta?.body?.error?.type === 'resource_already_exists_exception') return + throw error + } } -export function getElasticsearchConfig( - retryConfig: ElasticsearchRetryConfig = {} -): Required { - return { - ...DEFAULT_ELASTICSEARCH_CONFIG, - ...retryConfig +// allow_partial_search_results: false refuses failed shards; a search that timed out still +// answers with the hits that it found +export function assertCompleteSearch( + result: Pick, + index: estypes.Indices +): void { + if (result.timed_out || result._shards.failed > 0) { + throw new Error(`Incomplete search on index ${index}: the result is partial`) } } -export function getElasticsearchConnectionStats(): { - attempts: number - lastConnection: number - connected: boolean - isRetrying: boolean - isMonitoring: boolean -} { - const singleton = ElasticsearchClientSingleton.getInstance() - return singleton.getConnectionStats() +// Reads every match, page by page, on a point in time. For internal scans that must see all +// documents; a plain search stops at MAX_RESULT_WINDOW hits. +export async function searchAllPages( + client: Client, + request: Pick & { + sort?: estypes.SortCombinations[] + }, + pageSize: number = MAX_RESULT_WINDOW +): Promise[]> { + const { index, sort = [], ...rest } = request + const keepAlive = '1m' + let pitId = (await client.openPointInTime({ index, keep_alive: keepAlive })).id + const hits: estypes.SearchHit[] = [] + try { + let searchAfter: estypes.SortResults | undefined + for (;;) { + const page = await client.search({ + ...rest, + pit: { id: pitId, keep_alive: keepAlive }, + // _shard_doc makes the order total, so search_after never skips or repeats a hit + sort: [...sort, { _shard_doc: 'asc' }], + size: pageSize, + search_after: searchAfter, + track_total_hits: false, + allow_partial_search_results: false + }) + // a short page must mean "no more hits", never "a shard did not answer" + assertCompleteSearch(page, index) + pitId = page.pit_id ?? pitId + hits.push(...page.hits.hits) + if (page.hits.hits.length < pageSize) return hits + searchAfter = page.hits.hits[page.hits.hits.length - 1].sort + } + } finally { + // the point in time expires by itself; a failed close must not hide the result + await client.closePointInTime({ id: pitId }).catch(() => {}) + } +} + +// real-time read by id; a missing document is null, any other error (a missing index too) +// is thrown +export async function getOrNull( + client: Client, + index: string, + id: string +): Promise<{ source: T; seqNo: number; primaryTerm: number } | null> { + try { + const result = await client.get({ index, id }) + return { + source: result._source, + seqNo: result._seq_no, + primaryTerm: result._primary_term + } + } catch (error) { + const missingIndex = error?.meta?.body?.error?.type === 'index_not_found_exception' + if (isStatus(error, 404) && !missingIndex) return null + throw error + } } diff --git a/src/components/database/ElasticsearchNodeMetrics.ts b/src/components/database/ElasticsearchNodeMetrics.ts new file mode 100644 index 000000000..f13fee5b1 --- /dev/null +++ b/src/components/database/ElasticsearchNodeMetrics.ts @@ -0,0 +1,182 @@ +import { Client, estypes } from '@elastic/elasticsearch' +import { OceanNodeDBConfig } from '../../@types/OceanNode.js' +import type { NodeMetricsHourly, NodeMetricsSnapshot } from '../../@types/nodeMetrics.js' +import { MAX_RESULT_WINDOW } from './BaseDatabase.js' +import { NodeMetricsProvider } from './NodeMetricsDatabase.js' +import { + assertCompleteSearch, + createElasticsearchClientWithRetry, + deleteMatching, + ensureIndex, + isStatus, + searchAllPages +} from './ElasticsearchConfigHelper.js' +import { stateSchemas } from './ElasticSchemas.js' +import { + HOUR_MS, + HourlyRow, + SampleRow, + aggregateRows, + floorToHour, + hourlyRowToApi, + hourlyToRow, + snapshotToSampleRow +} from './nodeMetricsAggregate.js' + +const SAMPLES_INDEX = stateSchemas.metricsSamples.index +const HOURLY_INDEX = stateSchemas.metricsHourly.index + +type SampleDoc = SampleRow & { instanceId: string } +type HourlyDoc = HourlyRow & { instanceId: string } + +// samples: auto id; hourly rows: id = `${instanceId}:${hourStart}` +export class ElasticsearchNodeMetrics implements NodeMetricsProvider { + private constructor(private config: OceanNodeDBConfig) {} + + // resolved per call: the health monitor replaces the client after a reconnect + private client(): Promise { + return createElasticsearchClientWithRetry(this.config) + } + + static async create(config: OceanNodeDBConfig): Promise { + await ensureIndex(config, stateSchemas.metricsSamples) + await ensureIndex(config, stateSchemas.metricsHourly) + return new ElasticsearchNodeMetrics(config) + } + + async insertSample(instanceId: string, snapshot: NodeMetricsSnapshot): Promise { + const document: SampleDoc = { instanceId, ...snapshotToSampleRow(snapshot) } + const client = await this.client() + await client.index({ index: SAMPLES_INDEX, document, refresh: 'wait_for' }) + } + + async getPendingSampleHours(instanceId: string, beforeHour: number): Promise { + const client = await this.client() + // every pending sample: an hour that is not listed here is purged without a roll-up + const hits = await searchAllPages>(client, { + index: SAMPLES_INDEX, + query: olderThanQuery(instanceId, 'collectedAt', beforeHour), + _source: ['collectedAt'] + }) + const hours = new Set(hits.map((hit) => floorToHour(hit._source.collectedAt))) + return [...hours].sort((a, b) => a - b) + } + + // Averages the samples of the hour into one hourly row, then deletes them. The first + // roll-up of an hour wins: the row is created once, so the samples that a partial delete + // (Elasticsearch does not roll back) leaves are deleted by the next run, not counted again. + async rollupHour(instanceId: string, hourStart: number): Promise { + const client = await this.client() + const hits = await this.getHourSamples(instanceId, hourStart) + if (hits.length === 0) return false + const document: HourlyDoc = { + instanceId, + ...hourlyToRow( + aggregateRows( + hits.map((hit) => hit._source), + hourStart + ) + ) + } + let written = true + try { + await client.index({ + index: HOURLY_INDEX, + id: `${instanceId}:${hourStart}`, + document, + op_type: 'create', + refresh: 'wait_for' + }) + } catch (error) { + if (!isStatus(error, 409)) throw error + written = false + } + await deleteMatching(client, SAMPLES_INDEX, { + ids: { values: hits.map((hit) => hit._id) } + }) + return written + } + + async getPartialHour( + instanceId: string, + hourStart: number + ): Promise { + const hits = await this.getHourSamples(instanceId, hourStart) + if (hits.length === 0) return null + const agg = aggregateRows( + hits.map((hit) => hit._source), + hourStart + ) + agg.partial = true + return agg + } + + async purgeOldSamples(instanceId: string, olderThan: number): Promise { + const query = olderThanQuery(instanceId, 'collectedAt', olderThan) + return deleteMatching(await this.client(), SAMPLES_INDEX, query) + } + + async deleteHourlyOlderThan(instanceId: string, cutoff: number): Promise { + const query = olderThanQuery(instanceId, 'hourStart', cutoff) + return deleteMatching(await this.client(), HOURLY_INDEX, query) + } + + async getHourly( + instanceId: string, + from: number, + to: number, + limit: number + ): Promise { + const client = await this.client() + const result = await client.search({ + index: HOURLY_INDEX, + query: { + bool: { + filter: [ + { term: { instanceId } }, + { range: { hourStart: { gte: from, lte: to } } } + ] + } + }, + sort: [{ hourStart: { order: 'asc' } }], + size: limit + }) + return result.hits.hits.map((hit) => hourlyRowToApi(hit._source)) + } + + private async getHourSamples( + instanceId: string, + hourStart: number + ): Promise[]> { + const client = await this.client() + const result = await client.search({ + index: SAMPLES_INDEX, + query: hourRangeQuery(instanceId, hourStart), + // one hour holds 60 samples, one per minute + size: MAX_RESULT_WINDOW, + // the rollup deletes the samples of the hour: it must see all of them + allow_partial_search_results: false + }) + assertCompleteSearch(result, SAMPLES_INDEX) + return result.hits.hits + } +} + +function hourRangeQuery(instanceId: string, hourStart: number) { + return { + bool: { + filter: [ + { term: { instanceId } }, + { range: { collectedAt: { gte: hourStart, lt: hourStart + HOUR_MS } } } + ] + } + } +} + +function olderThanQuery(instanceId: string, field: string, cutoff: number) { + return { + bool: { + filter: [{ term: { instanceId } }, { range: { [field]: { lt: cutoff } } }] + } + } +} diff --git a/src/components/database/ElasticsearchStores.ts b/src/components/database/ElasticsearchStores.ts new file mode 100644 index 000000000..01fcc127f --- /dev/null +++ b/src/components/database/ElasticsearchStores.ts @@ -0,0 +1,40 @@ +import type { OceanNodeDBConfig } from '../../@types/OceanNode.js' +import { AuthTokenDatabase } from './AuthTokenDatabase.js' +import { C2DDatabase } from './C2DDatabase.js' +import type { DatabaseStores } from './DatabaseFactory.js' +import { + ElasticsearchAccessListDatabase, + ElasticsearchDdoDatabase, + ElasticsearchDdoStateDatabase, + ElasticsearchEscrowDatabase, + ElasticsearchIndexerDatabase, + ElasticsearchLogDatabase, + ElasticsearchNonceDatabase, + ElasticsearchOrderDatabase +} from './ElasticSearchDatabase.js' +import { ElasticsearchAuthToken } from './ElasticsearchAuthToken.js' +import { ElasticsearchBucketRegistry } from './ElasticsearchBucketRegistry.js' +import { ElasticsearchCompute } from './ElasticsearchCompute.js' +import { ElasticsearchConfigDatabase } from './ElasticsearchConfigDatabase.js' +import { ElasticsearchNodeMetrics } from './ElasticsearchNodeMetrics.js' + +// the stores are built one after the other, so the first failure stops the rest +export async function createElasticsearchStores( + config: OceanNodeDBConfig +): Promise { + return { + nonce: await ElasticsearchNonceDatabase.create(config), + config: await ElasticsearchConfigDatabase.create(config), + c2d: new C2DDatabase(config, await ElasticsearchCompute.create(config)), + authToken: new AuthTokenDatabase(config, await ElasticsearchAuthToken.create(config)), + buckets: await ElasticsearchBucketRegistry.create(config), + nodeMetrics: await ElasticsearchNodeMetrics.create(config), + ddo: await ElasticsearchDdoDatabase.create(config), + indexer: await ElasticsearchIndexerDatabase.create(config), + logs: await ElasticsearchLogDatabase.create(config), + order: await ElasticsearchOrderDatabase.create(config), + ddoState: await ElasticsearchDdoStateDatabase.create(config), + accessList: await ElasticsearchAccessListDatabase.create(config), + escrow: await ElasticsearchEscrowDatabase.create(config) + } +} diff --git a/src/components/database/NodeMetricsDatabase.ts b/src/components/database/NodeMetricsDatabase.ts new file mode 100644 index 000000000..cd2026c9e --- /dev/null +++ b/src/components/database/NodeMetricsDatabase.ts @@ -0,0 +1,24 @@ +import type { NodeMetricsHourly, NodeMetricsSnapshot } from '../../@types/nodeMetrics.js' + +// per-instance node metrics history: raw minute samples rolled up into hourly buckets. +// HA instances share one peer id, so every call is keyed by the instance id. +export interface NodeMetricsProvider { + insertSample(instanceId: string, snapshot: NodeMetricsSnapshot): Promise + // distinct floored hours with raw samples strictly before `beforeHour`, ascending + getPendingSampleHours(instanceId: string, beforeHour: number): Promise + // averages the hour's samples into one hourly row, then deletes them; false when none + rollupHour(instanceId: string, hourStart: number): Promise + // live, unstored average of an unrolled hour flagged partial; null when no samples + getPartialHour(instanceId: string, hourStart: number): Promise + // returns the number of samples deleted + purgeOldSamples(instanceId: string, olderThan: number): Promise + // returns the number of hourly rows deleted + deleteHourlyOlderThan(instanceId: string, cutoff: number): Promise + // hourStart in [from, to], ascending, at most `limit` rows + getHourly( + instanceId: string, + from: number, + to: number, + limit: number + ): Promise +} diff --git a/src/components/database/SQLLiteConfigDatabase.ts b/src/components/database/SQLLiteConfigDatabase.ts deleted file mode 100644 index c3b092912..000000000 --- a/src/components/database/SQLLiteConfigDatabase.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { DATABASE_LOGGER } from '../../utils/logging/common.js' -import { SQLiteProvider } from './sqlite.js' - -export class SQLLiteConfigDatabase { - private provider: SQLiteProvider - - constructor() { - return (async (): Promise => { - DATABASE_LOGGER.debug('Config Database initiated with SQLite provider') - - // SqliteClient creates the parent directory on construction. - this.provider = new SQLiteProvider('databases/config.sqlite') - await this.provider.createTableForConfig() - - return this - })() as unknown as SQLLiteConfigDatabase - } - - async createOrUpdateConfig(key: string = 'version', value: string) { - try { - return await this.provider.createOrUpdateConfig(key, value) - } catch (error) { - const errorMsg = `Error when creating new version entry ${value}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async retrieveValue(key: string = 'version') { - try { - return await this.provider.retrieveValue(key) - } catch (error) { - const errorMsg = `Error when retrieving latest version entry: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } -} diff --git a/src/components/database/SQLLiteNonceDatabase.ts b/src/components/database/SQLLiteNonceDatabase.ts deleted file mode 100644 index 5169bd8a4..000000000 --- a/src/components/database/SQLLiteNonceDatabase.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { OceanNodeDBConfig } from '../../@types/OceanNode.js' -import { DATABASE_LOGGER } from '../../utils/logging/common.js' -import { addressCasingVariants, normalizeAddress } from '../../utils/evmAddress.js' -import { AbstractNonceDatabase } from './BaseDatabase.js' -import { SQLiteProvider } from './sqlite.js' -import { TypesenseSchema } from './TypesenseSchemas.js' - -export class SQLLiteNonceDatabase extends AbstractNonceDatabase { - private provider: SQLiteProvider - - constructor(config: OceanNodeDBConfig, schema: TypesenseSchema) { - super(config, schema) - return (async (): Promise => { - DATABASE_LOGGER.debug('Nonce Database initiated with SQLite provider') - - // SqliteClient creates the parent directory on construction. - this.provider = new SQLiteProvider('databases/nonceDatabase.sqlite') - await this.provider.createTableForNonce() - - return this - })() as unknown as SQLLiteNonceDatabase - } - - async create(address: string, nonce: number) { - try { - return await this.provider.createNonce(normalizeAddress(address), nonce) - } catch (error) { - const errorMsg = - `Error when creating new nonce entry ${nonce} for address ${address}: ` + - error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - // Rows are keyed by the checksummed address. Pre-normalization rows may be keyed by another - // casing of the SAME address, so read every variant and keep the HIGHEST nonce: the row id - // may migrate, but the nonce must stay monotonic — dropping back to 0 would briefly re-open - // the replay window that the nonce exists to close. - async retrieve(address: string) { - const id = normalizeAddress(address) - try { - let highest: number | null = null - for (const variant of addressCasingVariants(address)) { - const row = await this.provider.retrieveNonce(variant) - if (row && row.nonce !== null && (highest === null || row.nonce > highest)) { - highest = row.nonce - } - } - return { id, nonce: highest } - } catch (error) { - const errorMsg = - `Error when retrieving nonce entry for address ${address}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async update(address: string, nonce: number) { - try { - return await this.provider.updateNonce(address, nonce) - } catch (error) { - const errorMsg = - `Error when updating nonce entry ${nonce} for address ${address}: ` + - error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async delete(address: string) { - try { - return await this.provider.deleteNonceEntry(address) - } catch (error) { - const errorMsg = - `Error when deleting nonce entry for address ${address}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } -} diff --git a/src/components/database/TypesenseDatabase.ts b/src/components/database/TypesenseDatabase.ts deleted file mode 100644 index 4e2d1daac..000000000 --- a/src/components/database/TypesenseDatabase.ts +++ /dev/null @@ -1,1138 +0,0 @@ -import { OceanNodeDBConfig } from '../../@types/OceanNode.js' -import { convertTypesenseConfig, Typesense, TypesenseError } from './typesense.js' -import { TypesenseSchema } from './TypesenseSchemas.js' -import { TypesenseSearchParams } from '../../@types/index.js' -import { LOG_LEVELS_STR } from '../../utils/logging/Logger.js' -import { safeStringify } from '../../utils/logging/redact.js' -import { DATABASE_LOGGER } from '../../utils/logging/common.js' - -import { ENVIRONMENT_VARIABLES, TYPESENSE_HITS_CAP } from '../../utils/constants.js' -import { - AbstractAccessListDatabase, - AbstractDdoDatabase, - AbstractDdoStateDatabase, - AbstractEscrowDatabase, - AbstractIndexerDatabase, - AbstractLogDatabase, - AbstractOrderDatabase -} from './BaseDatabase.js' -import { AccessListUser } from '../../@types/AccessList.js' -import { EscrowEvent } from '../../@types/Escrow.js' -import { buildTypesenseFilterBy } from './escrowSearch.js' -import { validateDDO } from '../../utils/asset.js' -import { DDOManager } from '@oceanprotocol/ddo-js' - -const LOG_INSERT_FAILED_KEY = 'typesense-log-insert-failed' -let logInsertFailing = false - -export class TypesenseOrderDatabase extends AbstractOrderDatabase { - private provider: Typesense - - constructor(config: OceanNodeDBConfig, schema: TypesenseSchema) { - super(config, schema) - return (async (): Promise => { - this.provider = new Typesense({ - ...convertTypesenseConfig(this.config.url), - logger: DATABASE_LOGGER - }) - try { - await this.provider.collections(this.getSchema().name).retrieve() - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - await this.provider.collections().create(this.getSchema()) - } else { - DATABASE_LOGGER.error( - `Typesense collection '${this.getSchema().name}' check failed: ${error.message}` - ) - } - } - return this - })() as unknown as TypesenseOrderDatabase - } - - getSchema(): TypesenseSchema { - return this.schema as TypesenseSchema - } - - async search( - query: Record, - maxResultsPerPage?: number, - pageNumber?: number - ) { - try { - let queryObj: TypesenseSearchParams - - // if queryObj is a string - if (typeof query === 'string') { - queryObj = JSON.parse(query) - } else { - queryObj = query as TypesenseSearchParams - } - - // Check if the necessary properties are present - if (!queryObj.q || !queryObj.query_by) { - throw new Error("The query object must include 'q' and 'query_by' properties.") - } - const maxPerPage = maxResultsPerPage - ? Math.min(maxResultsPerPage, TYPESENSE_HITS_CAP) - : TYPESENSE_HITS_CAP // Cap maxResultsPerPage at 250 - const page = pageNumber || 1 // Default to the first page if pageNumber is not provided - - // Modify the query to include pagination parameters - const searchParams: TypesenseSearchParams = { - ...queryObj, - per_page: maxPerPage, - page - } - - const result = await this.provider - .collections(this.getSchema().name) - .documents() - .search(searchParams) - - // Instead of pushing the entire result, only include the documents - return result.hits.map((hit) => hit.document) - } catch (error) { - const errorMsg = `Error when searching order entries: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async create( - orderId: string, - type: string, - timestamp: number, - consumer: string, - payer: string, - datatokenAddress: string, - nftAddress: string, - did: string, - startOrderId?: string - ) { - try { - return await this.provider.collections(this.getSchema().name).documents().create({ - id: orderId, - type, - timestamp, - consumer, - payer, - datatokenAddress, - nftAddress, - did, - startOrderId - }) - } catch (error) { - const errorMsg = - `Error when creating order entry ${orderId} at timestamp ${timestamp} by payer ${payer} for consumer ${consumer}: ` + - error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async retrieve(orderId: string) { - try { - return await this.provider - .collections(this.getSchema().name) - .documents() - .retrieve(orderId) - } catch (error) { - const errorMsg = `Error when retrieving order ${orderId}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async update( - orderId: string, - type: string, - timestamp: number, - consumer: string, - payer: string, - datatokenAddress?: string, - startOrderId?: string - ) { - try { - return await this.provider - .collections(this.getSchema().name) - .documents() - .update(orderId, { - type, - timestamp, - consumer, - payer, - datatokenAddress, - startOrderId - }) - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - return await this.provider - .collections(this.getSchema().name) - .documents() - .create({ id: orderId, type, timestamp, consumer, payer, startOrderId }) - } - const errorMsg = - `Error when updating order entry ${orderId} at timestamp ${timestamp} by payer ${payer} for consumer ${consumer}: ` + - error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async delete(orderId: string) { - try { - return await this.provider - .collections(this.getSchema().name) - .documents() - .delete(orderId) - } catch (error) { - const errorMsg = `Error when deleting order ${orderId}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } -} - -export class TypesenseEscrowDatabase extends AbstractEscrowDatabase { - private provider: Typesense - - constructor(config: OceanNodeDBConfig, schema: TypesenseSchema) { - super(config, schema) - return (async (): Promise => { - this.provider = new Typesense({ - ...convertTypesenseConfig(this.config.url), - logger: DATABASE_LOGGER - }) - try { - await this.provider.collections(this.getSchema().name).retrieve() - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - await this.provider.collections().create(this.getSchema()) - } else { - DATABASE_LOGGER.error( - `Typesense collection '${this.getSchema().name}' check failed: ${error.message}` - ) - } - } - return this - })() as unknown as TypesenseEscrowDatabase - } - - getSchema(): TypesenseSchema { - return this.schema as TypesenseSchema - } - - async create(event: EscrowEvent) { - try { - return await this.provider - .collections(this.getSchema().name) - .documents() - .create({ ...event }) - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 409) { - return { ...event } - } - const errorMsg = `Error when creating escrow event ${event.id}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async retrieve(id: string) { - try { - return await this.provider - .collections(this.getSchema().name) - .documents() - .retrieve(id) - } catch (error) { - const errorMsg = `Error when retrieving escrow event ${id}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async search(filters: Record, offset?: number, size?: number) { - try { - // Values may be equality scalars or RangeFilter objects (range on numeric fields). - const filterBy = buildTypesenseFilterBy(filters) - - // clamp the page size so a single request can't return an unbounded set - const limit = Math.min(size && size > 0 ? size : 100, TYPESENSE_HITS_CAP) - const from = offset && offset > 0 ? offset : 0 - - const searchParams: TypesenseSearchParams = { - q: '*', - query_by: 'eventType', - offset: from, - limit - } - if (filterBy) { - searchParams.filter_by = filterBy - } - - const result = await this.provider - .collections(this.getSchema().name) - .documents() - .search(searchParams) - - return result.hits.map((hit) => hit.document) - } catch (error) { - const errorMsg = - `Error when searching escrow events (filters ${safeStringify(filters, { maxLength: 200 })}): ` + - error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async delete(id: string) { - try { - return await this.provider.collections(this.getSchema().name).documents().delete(id) - } catch (error) { - const errorMsg = `Error when deleting escrow event ${id}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } -} - -export class TypesenseDdoStateDatabase extends AbstractDdoStateDatabase { - private provider: Typesense - - constructor(config: OceanNodeDBConfig, schema: TypesenseSchema) { - super(config, schema) - return (async (): Promise => { - this.provider = new Typesense({ - ...convertTypesenseConfig(this.config.url), - logger: DATABASE_LOGGER - }) - try { - await this.provider.collections(this.schema.name).retrieve() - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - await this.provider.collections().create(this.schema) - } else { - DATABASE_LOGGER.error( - `Typesense collection '${this.schema.name}' check failed: ${error.message}` - ) - } - } - return this - })() as unknown as TypesenseDdoStateDatabase - } - - async create( - chainId: number, - did: string, - nftAddress: string, - txId: string = ' ', - valid: boolean = true, - errorMsg: string = ' ' - ) { - try { - return await this.provider - .collections(this.schema.name) - .documents() - .create({ id: did, chainId, did, nft: nftAddress, txId, valid, error: errorMsg }) - } catch (error) { - const errorMsg = `Error when saving ddo state for: ${did} Error: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async retrieve(did: string) { - try { - return await this.provider.collections(this.schema.name).documents().retrieve(did) - } catch (error) { - const errorMsg = - `Error when retrieving the state of the ddo with id: ${did}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async search(query: Record) { - try { - const result = await this.provider - .collections(this.schema.name) - .documents() - .search(query as TypesenseSearchParams) - return result - } catch (error) { - const errorMsg = - `Error when searching ddo state (query ${safeStringify(query, { maxLength: 200 })}): ` + - error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async update( - chainId: number, - did: string, - nftAddress: string, - txId: string = ' ', - valid: boolean = true, - errorMsg: string = ' ' - ) { - try { - return await this.provider - .collections(this.schema.name) - .documents() - .update(did, { chainId, did, nft: nftAddress, txId, valid, error: errorMsg }) - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - return await this.provider.collections(this.schema.name).documents().create({ - id: did, - chainId, - did, - nft: nftAddress, - txId, - valid, - error: errorMsg - }) - } - const errorMessage = - `Error when saving ddo state for: ${did} Error: ` + error.message - DATABASE_LOGGER.error(errorMessage) - return null - } - } - - async delete(did: string) { - try { - return await this.provider.collections(this.schema.name).documents().delete(did) - } catch (error) { - const errorMsg = `Error when deleting ddo state ${did}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } -} - -export class TypesenseDdoDatabase extends AbstractDdoDatabase { - private provider: Typesense - - constructor(config: OceanNodeDBConfig, schemas: TypesenseSchema[]) { - super(config, schemas) - return (async (): Promise => { - this.provider = new Typesense({ - ...convertTypesenseConfig(this.config.url), - logger: DATABASE_LOGGER - }) - for (const ddoSchema of this.getSchemas()) { - try { - await this.provider.collections(ddoSchema.name).retrieve() - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - await this.provider.collections().create(ddoSchema) - } else { - DATABASE_LOGGER.error( - `Typesense collection '${ddoSchema.name}' check failed: ${error.message}` - ) - } - } - } - return this - })() as unknown as TypesenseDdoDatabase - } - - getSchemas(): TypesenseSchema[] { - return this.schemas as TypesenseSchema[] - } - - getDDOSchema(ddo: Record): TypesenseSchema { - // Find the schema based on the DDO version OR use the short DDO schema when state !== 0 - let schemaName: string - - const ddoInstance = DDOManager.getDDOClass(ddo) - const ddoData = ddoInstance.getDDOData() - if ('indexedMetadata' in ddoData && ddoData?.indexedMetadata?.nft.state !== 0) { - schemaName = 'op_ddo_short' - } else if (ddo.version) { - schemaName = `op_ddo_v${ddo.version}` - } - const schema = this.getSchemas().find((s) => s.name === schemaName) - DATABASE_LOGGER.debug(`Returning schema: ${schemaName}`) - return schema - } - - private async deleteDDOFromOtherSchemas( - did: string, - currentSchema: TypesenseSchema - ): Promise { - for (const schema of this.getSchemas()) { - if (schema.name === currentSchema.name) { - continue - } - - try { - await this.provider.collections(schema.name).documents().delete(did) - } catch (error) { - if (!(error instanceof TypesenseError && error.httpStatus === 404)) { - DATABASE_LOGGER.error( - `Error when deleting stale DDO entry ${did} from schema ${schema.name}: ` + - error.message - ) - } - } - } - } - - async search( - query: Record, - maxResultsPerPage?: number, - pageNumber?: number - ) { - try { - let queryObj: TypesenseSearchParams - // if queryObj is a string - if (typeof query === 'string') { - queryObj = JSON.parse(query) as TypesenseSearchParams - } else { - queryObj = query as TypesenseSearchParams - } - - const maxPerPage = maxResultsPerPage - ? Math.min(maxResultsPerPage, TYPESENSE_HITS_CAP) - : TYPESENSE_HITS_CAP // Cap maxResultsPerPage at 250 - const page = pageNumber || 1 // Default to the first page if pageNumber is not provided - const results = [] - - for (const schema of this.getSchemas()) { - // Extend the query with pagination parameters - const searchParams: TypesenseSearchParams = { - ...queryObj, - per_page: maxPerPage, - page - } - const result = await this.provider - .collections(schema.name) - .documents() - .search(searchParams) - results.push(result) - } - - return results - } catch (error) { - const errorMsg = - `Error when searching by query (query ${safeStringify(query, { maxLength: 200 })}): ` + - error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async create(ddo: Record) { - const schema = this.getDDOSchema(ddo) - if (!schema) { - throw new Error(`Schema for version ${ddo.version} not found`) - } - try { - // avoid failure because of schema - if (ddo?.indexedMetadata?.nft) delete ddo.nft - - const validation = await validateDDO(ddo) - if (validation === true) { - return await this.provider - .collections(schema.name) - .documents() - .create({ ...ddo }) - } else { - throw new Error(`Validation of DDO with schema version ${ddo.version} failed`) - } - } catch (error) { - const errorMsg = `Error when creating DDO entry ${ddo.id}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async retrieve(id: string): Promise { - let ddo = null - for (const schema of this.getSchemas()) { - try { - ddo = await this.provider.collections(schema.name).documents().retrieve(id) - if (ddo) { - break - } - } catch (error) { - if (!(error instanceof TypesenseError && error.httpStatus === 404)) { - // Log error other than not found - DATABASE_LOGGER.error( - `Error when retrieving DDO entry ${id} from schema ${schema.name}: ` + - error.message - ) - } - } - } - - if (!ddo) { - DATABASE_LOGGER.debug(`DDO entry with ID ${id} not found in any schema.`) - } - - return ddo - } - - async update(ddo: Record) { - const schema = this.getDDOSchema(ddo) - if (!schema) { - throw new Error(`Schema for version ${ddo.version} not found`) - } - try { - // avoid issue with nft fields, due to schema - if (ddo?.indexedMetadata?.nft) delete ddo.nft - const validation = await validateDDO(ddo) - if (validation === true) { - const response = await this.provider - .collections(schema.name) - .documents() - .upsert(ddo) - await this.deleteDDOFromOtherSchemas(ddo.id, schema) - return response - } else { - throw new Error( - `Validation of DDO with schema version ${ddo.version} failed with errors` - ) - } - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - // No DDO was found to update so we will create a new one. - // First we must delete the old version if it exist in another collection - await this.delete(ddo.id) - - return await this.create(ddo) - } - const errorMsg = `Error when updating DDO entry ${ddo.id}: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async delete(did: string) { - let isDeleted = false - for (const schema of this.getSchemas()) { - try { - const response = await this.provider - .collections(schema.name) - .documents() - .delete(did) - if (response.id === did) { - isDeleted = true - DATABASE_LOGGER.debug( - `Response for deleting the ddo: ${response.id}, isDeleted: ${isDeleted}` - ) - return response - } - } catch (error) { - if (!(error instanceof TypesenseError && error.httpStatus === 404)) { - // Log error other than not found - DATABASE_LOGGER.error( - `Error when deleting DDO entry ${did} from schema ${schema.name}: ` + - error.message - ) - } - } - } - - if (!isDeleted) { - DATABASE_LOGGER.debug( - `DDO entry with ID ${did} not found in any schema or could not be deleted.` - ) - } - } - - async deleteAllAssetsFromChain(chainId: number, batchSize?: number): Promise { - let numDeleted = 0 - for (const schema of this.getSchemas()) { - try { - const response = await this.provider - .collections(schema.name) - .documents() - .deleteByChainId(`chainId:${chainId}`, batchSize) - - DATABASE_LOGGER.debug( - `Number of deleted ddos on schema ${schema.name} : ${response.num_deleted}` - ) - - numDeleted += response.num_deleted - } catch (error) { - if (!(error instanceof TypesenseError && error.httpStatus === 404)) { - // Log error other than not found - DATABASE_LOGGER.error( - `Error when deleting DDOs from schema ${schema.name}: ` + error.message - ) - } - } - } - return numDeleted - } -} - -export class TypesenseIndexerDatabase extends AbstractIndexerDatabase { - private provider: Typesense - - // constant for the node version document ID - private static readonly VERSION_DOC_ID = 'node_version' - - constructor(config: OceanNodeDBConfig, schema: TypesenseSchema) { - super(config, schema) - return (async (): Promise => { - this.provider = new Typesense({ - ...convertTypesenseConfig(this.config.url), - logger: DATABASE_LOGGER - }) - try { - await this.provider.collections(this.schema.name).retrieve() - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - await this.provider.collections().create(this.schema) - } else { - DATABASE_LOGGER.error( - `Typesense collection '${this.schema.name}' check failed: ${error.message}` - ) - } - } - return this - })() as unknown as TypesenseIndexerDatabase - } - - async create(network: number, lastIndexedBlock: number) { - try { - return await this.provider - .collections(this.schema.name) - .documents() - .create({ id: network.toString(), lastIndexedBlock }) - } catch (error) { - const errorMsg = - `Error when creating indexer entry on network ${network.toString()} with last indexed block ${lastIndexedBlock}: ` + - error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async retrieve(network: number) { - try { - return await this.provider - .collections(this.schema.name) - .documents() - .retrieve(network.toString()) - } catch (error) { - const errorMsg = - `Error when retrieving indexer entry on network ${network.toString()}: ` + - error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async update(network: number, lastIndexedBlock: number) { - try { - return await this.provider - .collections(this.schema.name) - .documents() - .update(network.toString(), { lastIndexedBlock }) - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - return await this.provider - .collections(this.schema.name) - .documents() - .create({ id: network.toString(), lastIndexedBlock }) - } - const errorMsg = - `Error when updating indexer entry on network ${network.toString()} with last indexed block ${lastIndexedBlock}: ` + - error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async delete(network: number) { - try { - return await this.provider - .collections(this.schema.name) - .documents() - .delete(network.toString()) - } catch (error) { - const errorMsg = - `Error when deleting indexer entry on network ${network.toString()}: ` + - error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } -} - -export class TypesenseLogDatabase extends AbstractLogDatabase { - private provider: Typesense - - constructor(config: OceanNodeDBConfig, schema: TypesenseSchema) { - super(config, schema) - return (async (): Promise => { - this.provider = new Typesense({ - ...convertTypesenseConfig(this.config.url), - logger: DATABASE_LOGGER - }) - try { - await this.provider.collections(this.schema.name).retrieve() - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - try { - await this.provider.collections().create(this.schema) - } catch (creationError) { - DATABASE_LOGGER.error( - `Failed to create Typesense collection '${this.schema.name}': ${creationError.message}` - ) - } - } else { - DATABASE_LOGGER.error( - `Typesense collection '${this.schema.name}' check failed: ${error.message}` - ) - } - } - return this - })() as unknown as TypesenseLogDatabase - } - - async insertLog(logEntry: Record) { - try { - const result = await this.provider - .collections(this.schema.name) - .documents() - .create(logEntry) - if (logInsertFailing) { - logInsertFailing = false - DATABASE_LOGGER.resetThrottle(LOG_INSERT_FAILED_KEY) - DATABASE_LOGGER.info('Log DB writes resumed') - } - return result - } catch (error) { - const errorMsg = `Error when inserting log entry: ` + error.message - logInsertFailing = true - // one line per failed write would double the output during a log DB outage - DATABASE_LOGGER.throttled( - LOG_INSERT_FAILED_KEY, - LOG_LEVELS_STR.LEVEL_ERROR, - errorMsg - ) - return null - } - } - - async retrieveLog(id: string): Promise | null> { - try { - return await this.provider.collections(this.schema.name).documents().retrieve(id) - } catch (error) { - const errorMsg = `Error when retrieving log entry: ` + error.message - DATABASE_LOGGER.error(errorMsg) - return null - } - } - - async retrieveMultipleLogs( - startTime: Date, - endTime: Date, - maxLogs: number, - moduleName?: string, - level?: string, - page?: number - ): Promise[]> { - try { - let filterConditions = `timestamp:>=${startTime.getTime()} && timestamp:<${endTime.getTime()}` - if (moduleName) { - filterConditions += ` && moduleName:${moduleName}` - } - if (level) { - filterConditions += ` && level:${level}` - } - - if (page !== undefined && page !== null) { - return await this.fetchLogPage( - filterConditions, - page, - Math.min(maxLogs, TYPESENSE_HITS_CAP) - ) - } - - const allLogs: Record[] = [] - let currentPage = 1 // Typesense pages are 1-based - - while (allLogs.length < maxLogs) { - const perPage = Math.min(TYPESENSE_HITS_CAP, maxLogs - allLogs.length) - - const searchParameters = { - q: '*', - query_by: 'message,level,meta', - filter_by: filterConditions, - sort_by: 'timestamp:desc', - per_page: perPage, - page: currentPage - } - - const result = await this.provider - .collections(this.schema.name) - .documents() - .search(searchParameters) - - const hits = result.hits.map((hit) => hit.document) - allLogs.push(...hits) - - if (hits.length < perPage) break - currentPage++ - } - - return allLogs.slice(0, maxLogs) - } catch (error) { - const errorMsg = `Error when retrieving multiple log entries: ${error.message}` - DATABASE_LOGGER.error(errorMsg) - return [] - } - } - - private async fetchLogPage( - filterConditions: string, - page: number, - perPage: number - ): Promise[]> { - const searchParameters = { - q: '*', - query_by: 'message,level,meta', - filter_by: filterConditions, - sort_by: 'timestamp:desc', - per_page: perPage, - page - } - const result = await this.provider - .collections(this.schema.name) - .documents() - .search(searchParameters) - return result.hits.map((hit) => hit.document) - } - - async delete(logId: string): Promise { - if (!logId) { - throw new Error('Log ID is required for deletion.') - } - try { - await this.provider.collections(this.schema.name).documents().delete(logId) - DATABASE_LOGGER.debug(`Deleted log with ID: ${logId}`) - } catch (error) { - DATABASE_LOGGER.error(`Error when deleting log entry: ${error.message}`) - throw error - } - } - - async deleteOldLogs(): Promise { - const defaultLogRetention = '2592000000' // 30 days in milliseconds - const currentTime = new Date().getTime() - const xTime = parseInt( - ENVIRONMENT_VARIABLES.LOG_RETENTION_TIME.value || defaultLogRetention - ) - const deleteBeforeTime = new Date(currentTime - xTime) - - try { - const oldLogs = await this.retrieveMultipleLogs( - new Date(0), - deleteBeforeTime, - 200, - undefined, - undefined - ) - - if (oldLogs) { - for (const log of oldLogs) { - if (log.id) { - await this.delete(log.id) - } - } - } - return oldLogs.length - } catch (error) { - DATABASE_LOGGER.error(`Error when deleting old log entries: ${error.message}`) - } - } - - async getLogsCount(): Promise { - try { - const res = await this.provider.collections(this.schema.name).retrieve() - return res && res.num_documents ? res.num_documents : 0 - } catch (e) { - DATABASE_LOGGER.error('Unable to retrieve logs count: ' + e.message) - return 0 - } - } -} - -export class TypesenseAccessListDatabase extends AbstractAccessListDatabase { - private provider: Typesense - - constructor(config: OceanNodeDBConfig, schema: TypesenseSchema) { - super(config, schema) - return (async (): Promise => { - this.provider = new Typesense({ - ...convertTypesenseConfig(this.config.url), - logger: DATABASE_LOGGER - }) - try { - await this.provider.collections(this.schema.name).retrieve() - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - await this.provider.collections().create(this.schema) - } else { - DATABASE_LOGGER.error( - `Typesense collection '${this.schema.name}' check failed: ${error.message}` - ) - } - } - return this - })() as unknown as TypesenseAccessListDatabase - } - - private docId(chainId: number, contractAddress: string): string { - return `${chainId}-${contractAddress.toLowerCase()}` - } - - async create( - chainId: number, - contractAddress: string, - transferable: boolean, - block: number, - txId: string, - name?: string, - symbol?: string - ) { - const id = this.docId(chainId, contractAddress) - const lowerContract = contractAddress.toLowerCase() - try { - const existing: any = await this.retrieve(chainId, contractAddress) - const doc = { - id, - chainId, - contractAddress: lowerContract, - name, - symbol, - transferable, - users: existing?.users ?? [], - deploymentBlock: block, - deploymentTxId: txId - } - if (existing) { - return await this.provider - .collections(this.schema.name) - .documents() - .update(id, doc) - } - return await this.provider.collections(this.schema.name).documents().create(doc) - } catch (error) { - this.logError(`upserting access list ${id}`, error) - return null - } - } - - async retrieve(chainId: number, contractAddress: string) { - const id = this.docId(chainId, contractAddress) - try { - const doc: any = await this.provider - .collections(this.schema.name) - .documents() - .retrieve(id) - return stripId(doc) - } catch (error) { - if (error instanceof TypesenseError && error.httpStatus === 404) { - return null - } - this.logError(`retrieving access list ${id}`, error) - return null - } - } - - async addUser(chainId: number, contractAddress: string, user: AccessListUser) { - const id = this.docId(chainId, contractAddress) - const lowerContract = contractAddress.toLowerCase() - const normalized: AccessListUser = { ...user, wallet: user.wallet.toLowerCase() } - try { - const existing: any = await this.retrieve(chainId, contractAddress) - const users: AccessListUser[] = existing?.users ?? [] - const exists = users.some((u) => u.tokenId === normalized.tokenId) - const nextUsers = exists ? users : [...users, normalized] - if (existing) { - return await this.provider - .collections(this.schema.name) - .documents() - .update(id, { users: nextUsers }) - } - return await this.provider.collections(this.schema.name).documents().create({ - id, - chainId, - contractAddress: lowerContract, - transferable: false, - users: nextUsers - }) - } catch (error) { - this.logError(`adding user ${normalized.wallet} to access list ${id}`, error) - return null - } - } - - async removeUserByTokenId(chainId: number, contractAddress: string, tokenId: number) { - const id = this.docId(chainId, contractAddress) - try { - const existing: any = await this.retrieve(chainId, contractAddress) - if (!existing) return null - const nextUsers = (existing.users ?? []).filter( - (u: AccessListUser) => u.tokenId !== tokenId - ) - return await this.provider - .collections(this.schema.name) - .documents() - .update(id, { users: nextUsers }) - } catch (error) { - this.logError(`removing tokenId ${tokenId} from access list ${id}`, error) - return null - } - } - - async searchByWallet(wallet: string, chainId?: number): Promise { - const lowerWallet = wallet.toLowerCase() - try { - const filterParts = [`users.wallet:=${lowerWallet}`] - if (chainId !== undefined) filterParts.push(`chainId:=${chainId}`) - const result = await this.provider - .collections(this.schema.name) - .documents() - .search({ - q: '*', - query_by: 'contractAddress', - filter_by: filterParts.join(' && '), - per_page: 250 - }) - return (result.hits ?? []).map((h: any) => stripId(h.document)) - } catch (error) { - this.logError(`searching access lists by wallet ${lowerWallet}`, error) - return [] - } - } - - async delete(chainId: number, contractAddress: string) { - const id = this.docId(chainId, contractAddress) - try { - return await this.provider.collections(this.schema.name).documents().delete(id) - } catch (error) { - this.logError(`deleting access list ${id}`, error) - return null - } - } - - private logError(action: string, error: any) { - DATABASE_LOGGER.error(`Error when ${action}: ${error?.message ?? error}`) - } -} - -function stripId(doc: any): any { - if (!doc) return doc - const { id: _id, ...rest } = doc - return rest -} diff --git a/src/components/database/TypesenseDdoStateQuery.ts b/src/components/database/TypesenseDdoStateQuery.ts deleted file mode 100644 index e343c3392..000000000 --- a/src/components/database/TypesenseDdoStateQuery.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { IDdoStateQuery } from '../../@types/DDO/IDdoStateQuery.js' - -export class TypesenseDdoStateQuery implements IDdoStateQuery { - buildQuery(did?: string, nft?: string, txId?: string): Record { - let query: Record = {} - - if (did) { - query = { - q: did, - query_by: 'did' - } - } - - if (nft) { - query = { - q: nft, - query_by: 'nft' - } - } - - if (txId) { - query = { - q: txId, - query_by: 'txId' - } - } - - return query - } -} diff --git a/src/components/database/TypesenseMetadataQuery.ts b/src/components/database/TypesenseMetadataQuery.ts deleted file mode 100644 index 841016d49..000000000 --- a/src/components/database/TypesenseMetadataQuery.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { IMetadataQuery } from '../../@types/DDO/IMetadataQuery.js' -import { SearchQuery } from '../../@types/DDO/SearchQuery.js' - -export class TypesenseMetadataQuery implements IMetadataQuery { - buildQuery(searchQuery: SearchQuery): Record { - const { query } = searchQuery - - if (this.isTypesenseQuery(searchQuery)) { - return searchQuery - } - - const typesenseQuery: Record = { - q: '*', - filter_by: '', - num_hits: searchQuery.size || 10, - start: searchQuery.from || 0 - } - - const filters: string[] = [] - if (query.bool.filter) { - query.bool.filter.forEach((filter: any) => { - if (filter.term) { - const field = Object.keys(filter.term)[0] - const value = filter.term[field] - filters.push(`${field}:=${value}`) - } else if (filter.terms) { - const field = Object.keys(filter.terms)[0] - const values = filter.terms[field].join(',') - filters.push(`${field}:=[${values}]`) - } - }) - } - - if (query.bool && query.bool.must_not) { - query.bool.must_not.forEach((mustNot: any) => { - if (mustNot.term) { - const field = Object.keys(mustNot.term)[0] - const value = mustNot.term[field] - filters.push(`${field}:!=${value}`) - } else if (mustNot.terms) { - const field = Object.keys(mustNot.terms)[0] - const values = mustNot.terms[field].join(',') - filters.push(`${field}:!=[${values}]`) - } - }) - } - - if (filters.length > 0) { - typesenseQuery.filter_by = filters.join(' && ') - } - - if (searchQuery.sort) { - typesenseQuery.sort_by = Object.entries(searchQuery.sort) - .map(([field, direction]: [string, string]) => { - return `${field}:${direction}` - }) - .join(',') - } - - return typesenseQuery - } - - private isTypesenseQuery(query: any): boolean { - return query && query.filter_by !== undefined - } -} diff --git a/src/components/database/TypesenseSchemas.ts b/src/components/database/TypesenseSchemas.ts deleted file mode 100644 index 0c2380cdb..000000000 --- a/src/components/database/TypesenseSchemas.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { TypesenseCollectionCreateSchema } from '../../@types/index.js' -import fs from 'fs' -import path, { dirname, resolve } from 'path' -import { fileURLToPath } from 'url' -import { DATABASE_LOGGER } from '../../utils/logging/common.js' - -export function readJsonSchemas(): TypesenseCollectionCreateSchema[] { - const jsonDocuments: TypesenseCollectionCreateSchema[] = [] - const pathToSchemaDir: string = '../../../schemas' - const currentModulePath = fileURLToPath(import.meta.url) - - try { - const currentDirectory = dirname(currentModulePath) - const schemaFilePath = resolve(currentDirectory, pathToSchemaDir) - const jsonFiles = fs - .readdirSync(schemaFilePath) - .filter((file) => path.extname(file) === '.json') - - if (jsonFiles.length === 0) { - DATABASE_LOGGER.error( - `No JSON files found in the schemas directory ${schemaFilePath}.` - ) - return [] - } else { - jsonFiles.forEach((file) => { - // eslint-disable-next-line security/detect-non-literal-fs-filename - const fileData = fs.readFileSync(path.join(schemaFilePath, file), 'utf-8') - const jsonFile = JSON.parse(fileData.toString()) - jsonDocuments.push(jsonFile) - }) - return jsonDocuments - } - } catch (error) { - DATABASE_LOGGER.error(`Failed to load JSON schemas from ${pathToSchemaDir}: ${error}`) - } - return [] -} - -export type TypesenseSchema = TypesenseCollectionCreateSchema -export type TypesenseSchemas = { - ddoSchemas: TypesenseSchema[] - nonceSchemas: TypesenseSchema - c2dSchemas: TypesenseSchema - indexerSchemas: TypesenseSchema - logSchemas: TypesenseSchema - orderSchema: TypesenseSchema - ddoStateSchema: TypesenseSchema - accessListSchema: TypesenseSchema - escrowSchema: TypesenseSchema -} -const ddoSchemas = readJsonSchemas() -export const typesenseSchemas: TypesenseSchemas = { - ddoSchemas, - nonceSchemas: { - name: 'nonce', - enable_nested_fields: true, - fields: [{ name: 'nonce', type: 'int64' }] - }, - c2dSchemas: { - name: 'c2djobs', - enable_nested_fields: true, - fields: [ - // not really needed because it will be SQL Lite - { name: 'clusterHash', type: 'string', optional: false }, - { name: 'configlogURL', type: 'string', optional: false }, - { name: 'publishlogURL', type: 'string', optional: false }, - { name: 'algologURL', type: 'string', optional: false }, - { name: 'outputsURL', type: 'auto', optional: false }, - { name: 'stopRequested', type: 'bool', optional: false }, - { name: 'algorithm', type: 'auto', optional: false }, - { name: 'assets', type: 'auto', optional: false }, - { name: 'isRunning', type: 'bool', optional: false }, - { name: 'isStarted', type: 'bool', optional: false }, - { name: 'containerImage', type: 'string', optional: false } - ] - }, - indexerSchemas: { - name: 'indexer', - enable_nested_fields: true, - fields: [{ name: 'lastIndexedBlock', type: 'int64' }] - }, - logSchemas: { - name: 'logs', - enable_nested_fields: true, - fields: [ - { name: 'timestamp', type: 'int64', sort: true }, - { name: 'level', type: 'string' }, - { name: 'message', type: 'string' }, - { name: 'moduleName', type: 'string', optional: true }, - { name: 'meta', type: 'string', optional: true } - ] - }, - orderSchema: { - name: 'order', - enable_nested_fields: true, - fields: [ - { name: 'type', type: 'string', enum: ['startOrder', 'reuseOrder'] }, - { name: 'timestamp', type: 'int64' }, - { name: 'consumer', type: 'string' }, - { name: 'payer', type: 'string' }, - { name: 'datatokenAddress', type: 'string' }, - { name: 'nftAddress', type: 'string' }, - { name: 'did', type: 'string' }, - { - name: 'startOrderId', - type: 'string', - optional: true, - dependencies: { type: ['reuseOrder'] } - } - ] - }, - ddoStateSchema: { - name: 'state', - enable_nested_fields: true, - fields: [ - { name: 'chainId', type: 'int64' }, - { name: 'did', type: 'string' }, - { name: 'nft', type: 'string' }, - { name: 'txId', type: 'string' }, - { name: 'valid', type: 'bool' }, - { name: 'error', type: 'string' } - ] - }, - accessListSchema: { - name: 'access_list', - enable_nested_fields: true, - fields: [ - { name: 'chainId', type: 'int64' }, - { name: 'contractAddress', type: 'string' }, - { name: 'name', type: 'string', optional: true }, - { name: 'symbol', type: 'string', optional: true }, - { name: 'transferable', type: 'bool' }, - { name: 'users', type: 'object[]', optional: true }, - { name: 'users.wallet', type: 'string[]', optional: true, facet: true }, - { name: 'users.tokenId', type: 'int64[]', optional: true }, - { name: 'deploymentBlock', type: 'int64', optional: true }, - { name: 'deploymentTxId', type: 'string', optional: true } - ] - }, - escrowSchema: { - name: 'escrow', - enable_nested_fields: true, - fields: [ - { name: 'eventType', type: 'string', facet: true }, - { name: 'chainId', type: 'int64', facet: true }, - { name: 'contract', type: 'string' }, - { name: 'block', type: 'int64' }, - { name: 'txHash', type: 'string' }, - { name: 'payer', type: 'string', optional: true }, - { name: 'payee', type: 'string', optional: true }, - { name: 'token', type: 'string', optional: true }, - { name: 'jobId', type: 'string', optional: true }, - // uint256 values kept as raw strings to avoid precision loss - { name: 'amount', type: 'string', optional: true }, - { name: 'expiry', type: 'string', optional: true }, - // proof (Claimed event bytes) is stored but never filtered; skip indexing - // it so a large hex value can't hit Typesense's indexed-field length limit. - { name: 'proof', type: 'string', optional: true, index: false }, - { name: 'maxLockedAmount', type: 'string', optional: true }, - { name: 'maxLockSeconds', type: 'string', optional: true }, - { name: 'maxLockCounts', type: 'string', optional: true }, - // ReLock event fields (uint256 kept as raw strings) - { name: 'oldAmount', type: 'string', optional: true }, - { name: 'newAmount', type: 'string', optional: true }, - { name: 'newExpiry', type: 'string', optional: true }, - // Subsidy events (Subsidized / LockSponsored / SponsorRefunded) + Auth v2 expiry - { name: 'provider', type: 'string', optional: true }, - { name: 'subsidyAmount', type: 'string', optional: true }, - { name: 'bonusAmount', type: 'string', optional: true }, - { name: 'expiryTimestamp', type: 'string', optional: true }, - { name: 'reclaimable', type: 'bool', optional: true } - ] - } -} diff --git a/src/components/database/assetQuery.ts b/src/components/database/assetQuery.ts new file mode 100644 index 000000000..24906a1de --- /dev/null +++ b/src/components/database/assetQuery.ts @@ -0,0 +1,270 @@ +import { z } from 'zod' +import { + Condition, + MAX_RESULT_WINDOW, + Page, + QueryError, + Scalar, + SortSpec +} from './BaseDatabase.js' + +// The query of the node: a typed request that every database runs with its own fixed +// queries. It has the searches that the markets use, and nothing more. See the "DDO +// search" section of docs/database.md. + +// a DDO v4 path and the same field in a DDO v5 (`credentialSubject.`) +const inBoth = (path: string) => [path, `credentialSubject.${path}`] + +// the places of each searchable DDO field +export const DDO_FIELDS = { + id: ['id'], + version: ['version'], + chainId: inBoth('chainId'), + nftAddress: inBoth('nftAddress'), + owner: ['indexedMetadata.nft.owner'], + type: inBoth('metadata.type'), + serviceType: inBoth('services.type'), + datatokenAddress: inBoth('services.datatokenAddress'), + serviceEndpoint: inBoth('services.serviceEndpoint'), + tags: inBoth('metadata.tags'), + created: inBoth('metadata.created'), + nftState: ['indexedMetadata.nft.state'], + purgatory: ['indexedMetadata.purgatory.state'], + priceType: ['indexedMetadata.stats.prices.type'], + dataspace: ['credentialSubject.dataspace'], + trustedAlgorithmDid: inBoth('services.compute.publisherTrustedAlgorithms.did'), + trustedAlgorithmServiceId: inBoth( + 'services.compute.publisherTrustedAlgorithms.serviceId' + ), + trustedAlgorithmPublisher: inBoth( + 'services.compute.publisherTrustedAlgorithmPublishers' + ), + indexedMetadata: ['indexedMetadata'], + // the fields of the text search + text: [ + ...inBoth('metadata.name'), + 'metadata.description', + 'credentialSubject.metadata.description.value', + ...inBoth('metadata.author'), + ...inBoth('metadata.tags'), + 'indexedMetadata.stats.name', + 'indexedMetadata.stats.symbol', + 'indexedMetadata.nft.owner', + 'indexedMetadata.stats.datatokenAddress' + ] +} + +// the sort keys; `relevance` is the text score then `created` on Elasticsearch, `created` +// on SQL +export const SORT_FIELDS = { + created: ['indexedMetadata.event.block'], + orders: ['indexedMetadata.stats.orders'], + price: ['indexedMetadata.stats.prices.price'], + tokenSymbol: ['indexedMetadata.stats.symbol'] +} +export const RELEVANCE = 'relevance' + +// the nft state that the markets hide unless the query includes it +const UNLISTED_STATE = 5 +// the page size when the request has none +const QUERY_PAGE_SIZE = 1000 + +const strings = z.array(z.string()) + +// the fields of an `anyOf` group +const ANY_OF_FIELDS = { + ids: DDO_FIELDS.id, + owners: DDO_FIELDS.owner, + datatokenAddresses: DDO_FIELDS.datatokenAddress +} + +const assetFilter = z + .object({ + chainIds: z.array(z.number().int()).optional(), + versions: strings.optional(), + ids: strings.optional(), + excludeIds: strings.optional(), + excludeNftAddresses: strings.optional(), + owners: strings.optional(), + // each group holds when one of its lists matches; all groups must hold + anyOf: z + .array( + z + .object({ + ids: strings.optional(), + owners: strings.optional(), + datatokenAddresses: strings.optional() + }) + .strict() + ) + .optional(), + types: z.array(z.enum(['dataset', 'algorithm'])).optional(), + serviceTypes: z.array(z.enum(['access', 'compute'])).optional(), + datatokenAddresses: strings.optional(), + tags: strings.optional(), + text: z.string().max(512).optional(), + nftStates: z.array(z.number().int()).optional(), + includeUnlisted: z.boolean().optional(), + includePurgatory: z.boolean().optional(), + priceTypes: strings.optional(), + excludePriceTypes: strings.optional(), + createdAfter: z + .string() + .refine((value) => !Number.isNaN(Date.parse(value)), 'must be an ISO date') + .optional(), + serviceEndpoints: strings.optional(), + dataspace: z.string().optional(), + trustedAlgorithm: z + .object({ did: z.string(), serviceId: z.string() }) + .strict() + .optional(), + trustedAlgorithmPublisher: z.string().optional(), + indexedOnly: z.boolean().optional() + }) + .strict() + +export type AssetFilter = z.infer + +const paging = { + // starts at 1 + page: z.number().int().min(1).default(1), + size: z.number().int().min(0).max(MAX_RESULT_WINDOW).default(QUERY_PAGE_SIZE) +} + +const assetsRequest = z + .object({ + type: z.literal('assets'), + filter: assetFilter.default({}), + sort: z + .object({ + by: z.enum([RELEVANCE, ...Object.keys(SORT_FIELDS)] as [string, ...string[]]), + order: z.enum(['asc', 'desc']).default('desc') + }) + .strict() + .optional(), + ...paging + }) + .strict() + +const ordersRequest = z + .object({ + type: z.literal('orders'), + filter: z.union([ + z.object({ consumer: z.string() }).strict(), + z.object({ payer: z.string() }).strict() + ]), + ...paging + }) + .strict() + +const tagsRequest = z + .object({ + type: z.literal('tags'), + filter: assetFilter.default({}), + // the largest number of tags in the answer + size: z.number().int().min(1).max(MAX_RESULT_WINDOW).default(QUERY_PAGE_SIZE) + }) + .strict() + +const queryRequest = z.discriminatedUnion('type', [ + assetsRequest, + ordersRequest, + tagsRequest +]) + +export type QueryRequest = z.infer +export type AssetsRequest = z.infer + +// the request of a caller, checked; throws QueryError (HTTP 400) +export function parseQueryRequest(body: unknown): QueryRequest { + if (!isObject(body)) throw new QueryError('The query must be a JSON object') + if ('query' in body || 'from' in body || 'index' in body) { + throw new QueryError( + 'The query of the node is { type, filter, sort, page, size }: an Elasticsearch ' + + 'query body is not accepted. See the DDO search section of docs/database.md' + ) + } + const parsed = queryRequest.safeParse({ type: 'assets', ...body }) + if (!parsed.success) { + const issues = parsed.error.issues + .map((issue) => `${issue.path.join('.') || 'query'}: ${issue.message}`) + .join('; ') + throw new QueryError(`Invalid query: ${issues}`) + } + const request = parsed.data + if ('page' in request && request.page * request.size > MAX_RESULT_WINDOW) { + throw new QueryError(`A query reads the first ${MAX_RESULT_WINDOW} documents at most`) + } + return request +} + +export function pageOf(request: { page: number; size: number }): Page { + return { offset: (request.page - 1) * request.size, limit: request.size } +} + +export function sortOf(sort: AssetsRequest['sort']): SortSpec[] { + const { by, order } = sort ?? { by: 'created', order: 'desc' } + // SQL has no text score: there, relevance gives the newest first + if (by === RELEVANCE) + return [{ paths: SORT_FIELDS.created, order: 'desc', relevance: true }] + return [{ paths: SORT_FIELDS[by as keyof typeof SORT_FIELDS], order }] +} + +// the conditions of an asset filter; unset fields add none +export function assetConditions(filter: AssetFilter): Condition[] { + const f = DDO_FIELDS + const conditions: Condition[] = [] + const anyOf = (paths: string[], values: Scalar[] | undefined) => { + if (values) conditions.push({ paths, anyOf: values }) + } + const noneOf = (paths: string[], values: Scalar[] | undefined) => { + if (values) conditions.push({ paths, noneOf: values }) + } + + anyOf(f.chainId, filter.chainIds) + anyOf(f.version, filter.versions) + anyOf(f.id, filter.ids) + noneOf(f.id, filter.excludeIds) + noneOf(f.nftAddress, filter.excludeNftAddresses) + anyOf(f.owner, filter.owners) + anyOf(f.type, filter.types) + anyOf(f.serviceType, filter.serviceTypes) + anyOf(f.datatokenAddress, filter.datatokenAddresses) + anyOf(f.tags, filter.tags) + anyOf(f.priceType, filter.priceTypes) + noneOf(f.priceType, filter.excludePriceTypes) + anyOf(f.serviceEndpoint, filter.serviceEndpoints) + for (const group of filter.anyOf ?? []) { + const either = Object.entries(group).map(([field, values]) => ({ + paths: ANY_OF_FIELDS[field as keyof typeof ANY_OF_FIELDS], + anyOf: values + })) + if (either.length > 0) conditions.push({ either }) + } + anyOf(f.nftState, filter.nftStates) + if (!filter.includeUnlisted) noneOf(f.nftState, [UNLISTED_STATE]) + if (!filter.includePurgatory) noneOf(f.purgatory, [true]) + if (filter.createdAfter) { + conditions.push({ paths: f.created, range: { gte: filter.createdAfter } }) + } + if (filter.dataspace !== undefined) anyOf(f.dataspace, [filter.dataspace]) + if (filter.trustedAlgorithm) { + anyOf(f.trustedAlgorithmDid, [filter.trustedAlgorithm.did]) + anyOf(f.trustedAlgorithmServiceId, [filter.trustedAlgorithm.serviceId]) + } + if (filter.trustedAlgorithmPublisher !== undefined) { + anyOf(f.trustedAlgorithmPublisher, [filter.trustedAlgorithmPublisher]) + } + if (filter.indexedOnly) conditions.push({ paths: f.indexedMetadata, exists: true }) + const text = filter.text?.trim() + if (text) { + // each word in the text fields, or the whole text is a DID + const words: Condition = { paths: f.text, words: text.split(/\s+/) } + conditions.push({ either: [words, { paths: f.id, anyOf: [text] }] }) + } + return conditions +} + +function isObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} diff --git a/src/components/database/computeJobs.ts b/src/components/database/computeJobs.ts new file mode 100644 index 000000000..e90a64901 --- /dev/null +++ b/src/components/database/computeJobs.ts @@ -0,0 +1,120 @@ +// What all compute stores know about jobs and service jobs, whatever the backend. +import type { DBComputeJob } from '../../@types/C2D/C2D.js' +import { create256Hash } from '../../utils/crypt.js' +import type { ServiceJobExpected } from './C2DDatabase.js' +import { + ServiceStatusNumber, + SERVICE_STOPPED_STATUSES, + isStoppedLike, + type ServiceJob +} from '../../@types/C2D/ServiceOnDemand.js' + +// Every status before Expired is "active": the consumer paid for the resources for a TIME +// WINDOW, so the reservation holds from the moment the record is created (Starting) through +// the whole start pipeline, while Running/Restarting/Stopping, through Error (container died +// on its own) AND through an explicit Stopped — a stopped service can be restarted anytime on +// the same resources until expiresAt. Only expiry releases the reservation. +export const ACTIVE_SERVICE_STATUSES = [ + ServiceStatusNumber.Starting, + ServiceStatusNumber.Locking, + ServiceStatusNumber.PullImage, + ServiceStatusNumber.BuildImage, + ServiceStatusNumber.Claiming, + ServiceStatusNumber.Running, + ServiceStatusNumber.Restarting, + ServiceStatusNumber.Stopping, + ...SERVICE_STOPPED_STATUSES, + ServiceStatusNumber.Error +] + +// Running, Error and Stopped all still hold their paid reservation, so all must be swept once +// past expiresAt: the sweep is the ONLY place the reservation is released. Stopping is swept +// too: a process crash mid-stop persists that status and nothing else recovers it. +export const EXPIRABLE_SERVICE_STATUSES = [ + ServiceStatusNumber.Running, + ServiceStatusNumber.Error, + ...SERVICE_STOPPED_STATUSES, + ServiceStatusNumber.Stopping +] + +// The node-internal job fields that live in the unindexed body. +const SECONDS_PER_DAY = 24 * 60 * 60 + +// an image that no job used since this time (Unix seconds) is old +export function oldImageCutoff(retentionDays: number): number { + return Math.floor(Date.now() / 1000) - retentionDays * SECONDS_PER_DAY +} + +// a job from an older release has no jobIdHash +export function withJobIdHash(job: DBComputeJob): DBComputeJob { + if (!job.jobIdHash && job.jobId) job.jobIdHash = create256Hash(job.jobId) + return job +} + +export function jobToBody(job: DBComputeJob): any { + return { + clusterHash: job.clusterHash, + configlogURL: job.configlogURL, + publishlogURL: job.publishlogURL, + algologURL: job.algologURL, + outputsURL: job.outputsURL, + stopRequested: job.stopRequested, + algorithm: job.algorithm, + assets: job.assets, + isRunning: job.isRunning, + isStarted: job.isStarted, + containerImage: job.containerImage, + resources: job.resources, + isFree: job.isFree, + algoStartTimestamp: job.algoStartTimestamp, + algoStopTimestamp: job.algoStopTimestamp, + metadata: job.metadata, + additionalViewers: job.additionalViewers, + terminationDetails: job.terminationDetails, + payment: job.payment, + algoDuration: job.algoDuration, + queueMaxWaitTime: job.queueMaxWaitTime, + // a queued job pulls its image later, with the registry auth of the user + encryptedDockerRegistryAuth: job.encryptedDockerRegistryAuth, + output: job.output, + outputBucketId: job.outputBucketId, + jobIdHash: job.jobIdHash, + buildStartTimestamp: job.buildStartTimestamp, + buildStopTimestamp: job.buildStopTimestamp, + // Runtime metrics snapshot. It MUST round-trip through the body: it is what the owner + // reads back on COMPUTE_GET_STATUS, and it is also the previous-sample accumulator every + // next sample needs — without it persisted, every sample is a "first sample", so CPU % + // can never be computed (no delta) and the memory peak never accumulates. Still + // node-internal on the way out: omitDBComputeFieldsFromComputeJob strips it unless the + // verified owner asked for it, so it stays out of the escrow claim proof. + runtimeMetrics: job.runtimeMetrics + } +} + +// agreementIds are stored as given; match both the 0x-prefixed and the bare form +export function agreementIdVariants(agreementId: string): string[] { + return agreementId.startsWith('0x') + ? [agreementId, agreementId.slice(2)] + : [agreementId, '0x' + agreementId] +} + +// A best-effort patch of a service job (metrics, readiness) is written only when the +// lifecycle fields are still the ones that the caller saw. +export function matchesExpected(job: ServiceJob, expected: ServiceJobExpected): boolean { + return ( + job.owner === expected.owner && + job.clusterHash === expected.clusterHash && + job.status === expected.status && + job.containerId === expected.containerId + ) +} + +// The reservation is tied to PAYMENT: an Error or Stopped service whose payment was never +// claimed must not hold resources, or anyone could keep the GPU of a node for free with an +// empty escrow account. The statuses before Running keep the reservation without a claimTx. +export function holdsReservation(job: ServiceJob): boolean { + return ( + (job.status !== ServiceStatusNumber.Error && !isStoppedLike(job.status)) || + !!job.payment?.claimTx + ) +} diff --git a/src/components/database/connectionEvents.ts b/src/components/database/connectionEvents.ts new file mode 100644 index 000000000..3a1682d5c --- /dev/null +++ b/src/components/database/connectionEvents.ts @@ -0,0 +1,8 @@ +import EventEmitter from 'node:events' + +// a database backend emits these when it loses and restores its connection +export const DB_EVENTS = { + CONNECTION_LOST: 'db:connection:lost', + CONNECTION_RESTORED: 'db:connection:restored' +} as const +export const DB_CONNECTION_EVENTS = new EventEmitter() diff --git a/src/components/database/ddoSchemas.ts b/src/components/database/ddoSchemas.ts new file mode 100644 index 000000000..a7c0fbb84 --- /dev/null +++ b/src/components/database/ddoSchemas.ts @@ -0,0 +1,19 @@ +import { DDOManager, ddoElasticMappings } from '@oceanprotocol/ddo-js' + +// The groups of DDOs, one for each DDO version: an index on Elasticsearch. The node status +// lists them as its supported schemas. +// "op_ddo_short" is a node-side group for deprecated DDOs (nft state !== 0). +export const DDO_SHORT_SCHEMA = 'op_ddo_short' +export const DDO_SCHEMA_NAMES: string[] = [ + ...ddoElasticMappings.map((mapping: { index: string }) => mapping.index), + DDO_SHORT_SCHEMA +] + +// the group of a DDO; undefined for a DDO with no version +export function ddoSchemaName(ddo: Record): string | undefined { + const ddoData = DDOManager.getDDOClass(ddo).getDDOData() + if ('indexedMetadata' in ddoData && ddoData?.indexedMetadata?.nft?.state !== 0) { + return DDO_SHORT_SCHEMA + } + return ddo.version ? `op_ddo_v${ddo.version}` : undefined +} diff --git a/src/components/database/esConditions.ts b/src/components/database/esConditions.ts new file mode 100644 index 000000000..c809d1b79 --- /dev/null +++ b/src/components/database/esConditions.ts @@ -0,0 +1,135 @@ +import type { estypes } from '@elastic/elasticsearch' +import { addressCasingVariants } from '../../utils/evmAddress.js' +import type { Condition, Scalar, SortSpec } from './BaseDatabase.js' + +type Query = estypes.QueryDslQueryContainer + +// The DDO mappings (@oceanprotocol/ddo-js) do not name these fields, so Elasticsearch maps +// them itself: a text field, with a `.keyword` field that holds the exact value. +const DYNAMIC_TEXT = new Set([ + 'indexedMetadata.nft.owner', + 'indexedMetadata.stats.prices.type', + 'indexedMetadata.stats.prices.price', + 'indexedMetadata.stats.symbol', + 'credentialSubject.dataspace' +]) + +// Exact values are case sensitive here, and the stored case of an address is not known: +// an address matches in each of its case forms. +const ADDRESS_PATHS = new Set([ + 'nftAddress', + 'credentialSubject.nftAddress', + 'services.datatokenAddress', + 'credentialSubject.services.datatokenAddress', + 'indexedMetadata.nft.owner', + 'consumer', + 'payer' +]) + +// The DDO mappings make these objects `nested`: a query on a field below one of them runs +// in a `nested` query, or it never matches. The v5 trusted algorithms are not nested. +const NESTED_SCOPES = [ + 'services', + 'services.compute.publisherTrustedAlgorithms', + 'credentialSubject.services' +] + +const MATCH_ALL: Query = { match_all: {} } +const MATCH_NONE: Query = { match_none: {} } + +// the field that holds the exact value of a path +function exact(path: string): string { + return DYNAMIC_TEXT.has(path) ? `${path}.keyword` : path +} + +// The query on a path, inside the nested scopes of the path, innermost first. The DDO +// indices of one search have different mappings, so a scope that an index does not have +// is ignored there. +function inScopes(path: string, query: Query): Query { + return NESTED_SCOPES.filter((scope) => path.startsWith(`${scope}.`)) + .sort((a, b) => b.length - a.length) + .reduce( + (inner, scope) => ({ + nested: { path: scope, query: inner, ignore_unmapped: true } + }), + query + ) +} + +function exactValues(path: string, values: Scalar[]): Scalar[] { + if (!ADDRESS_PATHS.has(path)) return values + return [...new Set(values.flatMap((value) => addressCasingVariants(String(value))))] +} + +// one path of the field must hold +function onAnyPath(paths: string[], make: (path: string) => Query): Query { + return paths.length === 1 + ? make(paths[0]) + : { bool: { should: paths.map(make), minimum_should_match: 1 } } +} + +// one fixed clause for each kind of condition +export function esCondition(c: Condition): Query { + if ('either' in c) { + if (c.either.length === 0) return MATCH_NONE + return { bool: { should: c.either.map(esCondition), minimum_should_match: 1 } } + } + const terms = (values: Scalar[]) => (path: string) => + inScopes(path, { terms: { [exact(path)]: exactValues(path, values) } }) + if ('anyOf' in c) { + return c.anyOf.length === 0 ? MATCH_NONE : onAnyPath(c.paths, terms(c.anyOf)) + } + if ('noneOf' in c) { + if (c.noneOf.length === 0) return MATCH_ALL + return { bool: { must_not: c.paths.map(terms(c.noneOf)) } } + } + if ('range' in c) { + return onAnyPath(c.paths, (path) => inScopes(path, { range: { [path]: c.range } })) + } + if ('exists' in c) { + return onAnyPath(c.paths, (path) => inScopes(path, { exists: { field: path } })) + } + // each word, as a word or the start of a word, in one of the fields + return { + bool: { + filter: c.words.map((word) => ({ + multi_match: { + query: word, + fields: c.paths, + type: 'phrase_prefix', + lenient: true + } + })) + } + } +} + +export function esQuery(conditions: Condition[]): Query { + return conditions.length > 0 + ? { bool: { filter: conditions.map(esCondition) } } + : MATCH_ALL +} + +// `tieBreaker` is a keyword field with one value for each document, for a stable page +export function esSort( + sort: SortSpec[], + tieBreaker?: string +): estypes.SortCombinations[] { + const keys = sort.flatMap(({ paths, order, relevance }): estypes.SortCombinations[] => { + const path = paths[0] + const byPath = { + [exact(path)]: { + order, + missing: '_last' as const, + unmapped_type: DYNAMIC_TEXT.has(path) ? 'keyword' : 'long' + } + } + // the text score first; equal scores (no text: all) in the order of the path + return relevance ? [{ _score: { order: 'desc' } }, byPath] : [byPath] + }) + return tieBreaker ? [...keys, { [tieBreaker]: { order: 'asc' } }] : keys +} + +export function totalOf(total: number | estypes.SearchTotalHits | undefined): number { + return typeof total === 'number' ? total : (total?.value ?? 0) +} diff --git a/src/components/database/escrowSearch.ts b/src/components/database/escrowSearch.ts index b03b553b9..77ebaaa98 100644 --- a/src/components/database/escrowSearch.ts +++ b/src/components/database/escrowSearch.ts @@ -1,6 +1,7 @@ import { RangeFilter } from '../../@types/commands.js' +import { Condition, DEFAULT_PAGE_SIZE } from './BaseDatabase.js' -// Fields stored as native numeric types in both backends (Typesense int64 / ES long). +// Fields stored as native numeric types (ES long). // Range filters on these are pushed down to the store's query DSL. export const NATIVE_RANGE_FIELDS = ['block', 'chainId'] @@ -95,7 +96,7 @@ export function validateEscrowRanges(ranges?: Record): { return { valid: true } } -// Native range fields are numeric in the store (Typesense int64 / ES long), so their bounds go +// Native range fields are numeric in the store (ES long), so their bounds go // straight into the numeric DSL. Emit each defined bound as a canonical decimal string so a // value given as hex ("0x64"), a number, or with leading zeros ("007") is not sent verbatim. // Absent bounds stay absent; a non-parseable bound (shouldn't occur post-validation) is left @@ -149,33 +150,41 @@ export function passesClientRanges( return true } -// Build a Typesense `filter_by` string from a filters map whose values are either equality -// scalars or RangeFilter objects. Range objects only make sense on numeric fields. -export function buildTypesenseFilterBy(filters: Record): string { - const clauses: string[] = [] - // An array value means several constraints on the SAME field, all ANDed — e.g. an equality - // and a range on `chainId`; each element is emitted as its own clause. - const emit = (field: string, value: any) => { +const MAX_PAGE_SIZE = 250 + +// the page of an escrow search; the size has a limit so that one request cannot read all +export function escrowPage( + offset?: number, + size?: number +): { from: number; limit: number } { + return { + from: offset && offset > 0 ? offset : 0, + limit: Math.min(size && size > 0 ? size : DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE) + } +} + +// The same filters map as conditions (for the SQL stores): equality, or a range for a +// RangeFilter object. An array value is several conditions on the same field. +export function escrowConditions(filters: Record): Condition[] { + const conditions: Condition[] = [] + const add = (field: string, value: any) => { if (value === undefined || value === null || value === '') return - if (isRangeFilter(value)) { - if (hasValue(value.min)) clauses.push(`${field}:>=${value.min}`) - if (hasValue(value.max)) clauses.push(`${field}:<=${value.max}`) - if (hasValue(value.gt)) clauses.push(`${field}:>${value.gt}`) - if (hasValue(value.lt)) clauses.push(`${field}:<${value.lt}`) - } else if (typeof value === 'string') { - // Backtick string values so spaces/special chars can't break the syntax; - // strip backticks from the value so it can't escape the quoted literal. - clauses.push(`${field}:=\`${value.replace(/`/g, '')}\``) - } else { - clauses.push(`${field}:=${value}`) + if (!isRangeFilter(value)) { + conditions.push({ paths: [field], anyOf: [value] }) + return } + const range: Record = {} + if (hasValue(value.min)) range.gte = value.min + if (hasValue(value.max)) range.lte = value.max + if (hasValue(value.gt)) range.gt = value.gt + if (hasValue(value.lt)) range.lt = value.lt + if (Object.keys(range).length) conditions.push({ paths: [field], range }) } for (const [field, value] of Object.entries(filters || {})) { - if (value === undefined || value === null || value === '') continue - if (Array.isArray(value)) value.forEach((v) => emit(field, v)) - else emit(field, value) + if (Array.isArray(value)) value.forEach((v) => add(field, v)) + else add(field, value) } - return clauses.join(' && ') + return conditions } // Build an Elasticsearch query body from the same filters map: `term` for equality, diff --git a/src/components/database/index.ts b/src/components/database/index.ts index df14eee1f..b949b2343 100644 --- a/src/components/database/index.ts +++ b/src/components/database/index.ts @@ -1,5 +1,4 @@ import { OceanNodeDBConfig } from '../../@types/OceanNode.js' -import { hasValidDBConfiguration } from '../../utils/database.js' import { configureCustomDBTransport, getLoggingConfig, @@ -15,188 +14,74 @@ import { AbstractEscrowDatabase, AbstractIndexerDatabase, AbstractLogDatabase, + AbstractNonceDatabase, AbstractOrderDatabase } from './BaseDatabase.js' import { C2DDatabase } from './C2DDatabase.js' -import { DatabaseFactory } from './DatabaseFactory.js' -import { ElasticsearchSchema } from './ElasticSchemas.js' -import { SQLLiteConfigDatabase } from './SQLLiteConfigDatabase.js' -import { SQLLiteNonceDatabase } from './SQLLiteNonceDatabase.js' -import { TypesenseSchema } from './TypesenseSchemas.js' +import { DatabaseFactory, DatabaseStores } from './DatabaseFactory.js' +import { ConfigDatabase } from './ConfigDatabase.js' import { AuthTokenDatabase } from './AuthTokenDatabase.js' -import { NodeMetricsDatabase } from './sqliteNodeMetrics.js' +import { NodeMetricsProvider } from './NodeMetricsDatabase.js' +import { BucketRegistry } from './BucketRegistry.js' -export type Schema = ElasticsearchSchema | TypesenseSchema +export type { DdoSchema } from './BaseDatabase.js' -export class Database { +function logWhereLogsGo(): void { + if (USE_DB_TRANSPORT()) { + return + } + if (isFileLoggingActive()) { + DATABASE_LOGGER.info( + `LOG_DB disabled; logs are written to disk (${getLoggingConfig().dir})` + ) + } else { + DATABASE_LOGGER.warn('LOG_DB and file logging are disabled: logs are not persisted') + } +} + +export class Database implements DatabaseStores { ddo: AbstractDdoDatabase - nonce: SQLLiteNonceDatabase + nonce: AbstractNonceDatabase indexer: AbstractIndexerDatabase logs: AbstractLogDatabase order: AbstractOrderDatabase ddoState: AbstractDdoStateDatabase accessList: AbstractAccessListDatabase escrow: AbstractEscrowDatabase - sqliteConfig: SQLLiteConfigDatabase + config: ConfigDatabase c2d: C2DDatabase authToken: AuthTokenDatabase - nodeMetrics: NodeMetricsDatabase - // true only when the metadata (Typesense/Elasticsearch) databases were all initialized. - // false in DEGRADED mode (metadata DB unreachable) and when no metadata DB is configured. - metadataInitialized: boolean = false - - constructor(private config: OceanNodeDBConfig) {} + buckets: BucketRegistry + nodeMetrics: NodeMetricsProvider - // When a metadata (Typesense/Elasticsearch) database fails to initialize we either give up - // (return null, so the caller keeps retrying while the DB comes up) or, when allowPartial is - // set, return the already-built SQLite core (nonce/config/c2d/authToken) so the node can run - // in DEGRADED mode with C2D / nonce / auth available but Indexer and DDO features disabled. - private static handleMetadataInitFailure( - db: Database, - allowPartial: boolean - ): Database | null { - if (allowPartial) { - // src/index.ts reports DEGRADED mode to the operator - DATABASE_LOGGER.debug( - 'Metadata database unreachable — starting in DEGRADED mode: Indexer and DDO features are disabled, C2D / nonce / auth remain available' - ) - return db - } - return null - } - - // a retry follows unless this is the final (allowPartial) attempt - private static logMetadataInitFailure( - name: string, - error: unknown, - allowPartial: boolean - ): void { - DATABASE_LOGGER.error( - `${name} database initialization failed${allowPartial ? '' : ' (will retry)'}: ${errorSummary(error)}` - ) + private constructor( + private dbConfig: OceanNodeDBConfig, + stores: DatabaseStores + ) { + Object.assign(this, stores) } - static async init( - config: OceanNodeDBConfig, - allowPartial: boolean = false - ): Promise { - const db = new Database(config) + // returns null when a store fails, so the caller retries while the database comes up + static async init(config: OceanNodeDBConfig): Promise { + logWhereLogsGo() try { - db.nonce = await DatabaseFactory.createNonceDatabase(config) - } catch (error) { - DATABASE_LOGGER.error(`Nonce database initialization failed: ${error}`) - return null - } - try { - db.sqliteConfig = await DatabaseFactory.createConfigDatabase() - } catch (error) { - DATABASE_LOGGER.error(`Config database initialization failed: ${error}`) - return null - } - try { - db.c2d = await DatabaseFactory.createC2DDatabase(config) - } catch (error) { - DATABASE_LOGGER.error(`C2D database initialization failed: ${error}`) - return null - } - try { - db.authToken = await DatabaseFactory.createAuthTokenDatabase(config) - } catch (error) { - DATABASE_LOGGER.error(`Auth database initialization failed: ${error}`) - return null - } - try { - db.nodeMetrics = await DatabaseFactory.createNodeMetricsDatabase() - } catch (error) { - // Node metrics history is a best-effort, non-critical feature: a failure here must not - // stop the node from starting. Leave db.nodeMetrics undefined; the cron jobs and history - // handler guard on its presence. - DATABASE_LOGGER.error(`Node metrics database initialization failed: ${error}`) - } - - if (hasValidDBConfiguration(config)) { - if (!USE_DB_TRANSPORT()) { - if (isFileLoggingActive()) { - DATABASE_LOGGER.info( - `LOG_DB disabled; logs are written to disk (${getLoggingConfig().dir})` - ) - } else { - DATABASE_LOGGER.warn( - 'LOG_DB and file logging are disabled: logs are not persisted' - ) - } - } - try { - db.ddo = await DatabaseFactory.createDdoDatabase(config) - } catch (error) { - Database.logMetadataInitFailure('DDO', error, allowPartial) - return Database.handleMetadataInitFailure(db, allowPartial) - } - try { - db.indexer = await DatabaseFactory.createIndexerDatabase(config) - } catch (error) { - Database.logMetadataInitFailure('Indexer', error, allowPartial) - return Database.handleMetadataInitFailure(db, allowPartial) - } - - try { - db.logs = await DatabaseFactory.createLogDatabase(config) - } catch (error) { - Database.logMetadataInitFailure('Logs', error, allowPartial) - return Database.handleMetadataInitFailure(db, allowPartial) - } - // attach the DB log transport only once the log database exists; a later call (retry) - // rebinds it to the new instance + const db = new Database(config, await DatabaseFactory.createStores(config)) + // a later call (retry) binds the log transport to the new log store if (USE_DB_TRANSPORT()) { configureCustomDBTransport(db, DATABASE_LOGGER) } - - try { - db.order = await DatabaseFactory.createOrderDatabase(config) - } catch (error) { - Database.logMetadataInitFailure('Order', error, allowPartial) - return Database.handleMetadataInitFailure(db, allowPartial) - } - - try { - db.ddoState = await DatabaseFactory.createDdoStateDatabase(config) - } catch (error) { - Database.logMetadataInitFailure('DDO State', error, allowPartial) - return Database.handleMetadataInitFailure(db, allowPartial) - } - - try { - db.accessList = await DatabaseFactory.createAccessListDatabase(config) - } catch (error) { - Database.logMetadataInitFailure('AccessList', error, allowPartial) - return Database.handleMetadataInitFailure(db, allowPartial) - } - - try { - db.escrow = await DatabaseFactory.createEscrowDatabase(config) - } catch (error) { - Database.logMetadataInitFailure('Escrow', error, allowPartial) - return Database.handleMetadataInitFailure(db, allowPartial) - } - // All metadata databases initialized successfully — this is a full (non-degraded) node. - db.metadataInitialized = true - } else { - // src/index.ts reports the missing / invalid DB_URL to the operator - DATABASE_LOGGER.debug( - 'Invalid DB URL. Only Nonce, C2D, Auth Token and Config Databases are initialized.' + DATABASE_LOGGER.info(`Database ready: ${config.dbType}`) + return db + } catch (error) { + DATABASE_LOGGER.error( + `Database initialization failed (will retry): ${errorSummary(error)}` ) + return null } - - DATABASE_LOGGER.info( - `Database ready: SQLite (nonce, config, c2d, auth${ - db.nodeMetrics ? ', metrics' : '' - })${db.metadataInitialized ? ` + ${config.dbType} metadata DB` : ', no metadata DB'}` - ) - return db } // useful to know which configuration was passed to DB getConfig(): OceanNodeDBConfig { - return this.config + return this.dbConfig } } diff --git a/src/components/database/nodeMetricsAggregate.ts b/src/components/database/nodeMetricsAggregate.ts new file mode 100644 index 000000000..24d740da7 --- /dev/null +++ b/src/components/database/nodeMetricsAggregate.ts @@ -0,0 +1,235 @@ +import type { + NodeMetricsEnvResource, + NodeMetricsGpu, + NodeMetricsHourly, + NodeMetricsSnapshot +} from '../../@types/nodeMetrics.js' + +// Rollup math for node metrics, kept in JavaScript (no Elasticsearch aggregations) so the +// stored rollup and the live partial hour give the same numbers. + +export const HOUR_MS = 60 * 60 * 1000 + +export function floorToHour(epochMs: number): number { + return Math.floor(epochMs / HOUR_MS) * HOUR_MS +} + +// Raw sample row as stored (scalars flattened, structured fields as JSON strings). +export interface SampleRow { + collectedAt: number + cpuUsagePercent: number + coresAllocated: number + hostCores: number + throttledCount: number + memUsedBytes: number + memLimitBytes: number + hostFreeBytes: number + hostTotalBytes: number + diskUsedBytes: number + rxBytes: number + txBytes: number + jobsRunning: number + jobsRunningFree: number + jobsQueued: number + jobsQueuedFree: number + sampledContainers: number + gpu: string + env: string +} + +// Hourly row as stored. +export interface HourlyRow extends Omit { + hourStart: number + sampleCount: number +} + +function safeParse(json: string | null | undefined, fallback: T): T { + if (!json) return fallback + try { + return JSON.parse(json) as T + } catch { + return fallback + } +} + +// a snapshot has every field of an hourly bucket +type MetricsSource = Pick< + NodeMetricsHourly, + 'cpu' | 'memory' | 'disk' | 'network' | 'jobs' | 'meta' | 'gpu' | 'env' +> + +// the fields that a sample row and an hourly row share (structured fields as JSON strings) +function metricsToRow(m: MetricsSource): Omit { + return { + cpuUsagePercent: m.cpu.usagePercent, + coresAllocated: m.cpu.coresAllocated, + hostCores: m.cpu.hostCores, + throttledCount: m.cpu.throttledCount, + memUsedBytes: m.memory.usedBytes, + memLimitBytes: m.memory.limitBytes, + hostFreeBytes: m.memory.hostFreeBytes, + hostTotalBytes: m.memory.hostTotalBytes, + diskUsedBytes: m.disk.usedBytes, + rxBytes: m.network.rxBytes, + txBytes: m.network.txBytes, + jobsRunning: m.jobs.running, + jobsRunningFree: m.jobs.runningFree, + jobsQueued: m.jobs.queued, + jobsQueuedFree: m.jobs.queuedFree, + sampledContainers: m.meta.sampledContainers, + gpu: JSON.stringify(m.gpu ?? []), + env: JSON.stringify(m.env ?? []) + } +} + +export function snapshotToSampleRow(snapshot: NodeMetricsSnapshot): SampleRow { + return { collectedAt: snapshot.collectedAt, ...metricsToRow(snapshot) } +} + +export function hourlyToRow(h: NodeMetricsHourly): HourlyRow { + return { hourStart: h.hourStart, sampleCount: h.sampleCount, ...metricsToRow(h) } +} + +// Averages a set of raw sample rows into one NodeMetricsHourly bucket. Pure (no DB writes) so +// both the stored roll-up and the live partial-hour produce byte-for-byte identical shapes. +export function aggregateRows( + samples: SampleRow[], + hourStart: number +): NodeMetricsHourly { + const n = samples.length + const mean = (pick: (s: SampleRow) => number): number => + samples.reduce((acc, s) => acc + (Number(pick(s)) || 0), 0) / n + return { + hourStart, + sampleCount: n, + cpu: { + usagePercent: mean((s) => s.cpuUsagePercent), + coresAllocated: mean((s) => s.coresAllocated), + hostCores: mean((s) => s.hostCores), + throttledCount: mean((s) => s.throttledCount) + }, + memory: { + usedBytes: mean((s) => s.memUsedBytes), + limitBytes: mean((s) => s.memLimitBytes), + hostFreeBytes: mean((s) => s.hostFreeBytes), + hostTotalBytes: mean((s) => s.hostTotalBytes) + }, + disk: { usedBytes: mean((s) => s.diskUsedBytes) }, + network: { rxBytes: mean((s) => s.rxBytes), txBytes: mean((s) => s.txBytes) }, + jobs: { + running: mean((s) => s.jobsRunning), + runningFree: mean((s) => s.jobsRunningFree), + queued: mean((s) => s.jobsQueued), + queuedFree: mean((s) => s.jobsQueuedFree) + }, + gpu: averageGpu(samples.map((s) => safeParse(s.gpu, []))), + env: averageEnv(samples.map((s) => safeParse(s.env, []))), + meta: { sampledContainers: mean((s) => s.sampledContainers) } + } +} + +export function hourlyRowToApi(r: HourlyRow): NodeMetricsHourly { + return { + hourStart: Number(r.hourStart), + sampleCount: Number(r.sampleCount), + cpu: { + usagePercent: Number(r.cpuUsagePercent), + coresAllocated: Number(r.coresAllocated), + hostCores: Number(r.hostCores), + throttledCount: Number(r.throttledCount) + }, + memory: { + usedBytes: Number(r.memUsedBytes), + limitBytes: Number(r.memLimitBytes), + hostFreeBytes: Number(r.hostFreeBytes), + hostTotalBytes: Number(r.hostTotalBytes) + }, + disk: { usedBytes: Number(r.diskUsedBytes) }, + network: { rxBytes: Number(r.rxBytes), txBytes: Number(r.txBytes) }, + jobs: { + running: Number(r.jobsRunning), + runningFree: Number(r.jobsRunningFree), + queued: Number(r.jobsQueued), + queuedFree: Number(r.jobsQueuedFree) + }, + gpu: safeParse(r.gpu, []), + env: safeParse(r.env, []), + meta: { sampledContainers: Number(r.sampledContainers) } + } +} + +// Average each numeric GPU field per resourceId across every sample that reported that device. +// A missing metric on a given sample simply does not contribute to that field's mean. +function averageGpu(perSample: NodeMetricsGpu[][]): NodeMetricsGpu[] { + const acc = new Map< + string, + { + vendor?: string + sums: Record + counts: Record + } + >() + const numericFields: (keyof NodeMetricsGpu)[] = [ + 'utilizationPercent', + 'memoryUsedBytes', + 'memoryTotalBytes', + 'temperatureC', + 'powerWatts' + ] + for (const list of perSample) { + for (const g of list ?? []) { + const id = String(g.resourceId) + if (!acc.has(id)) acc.set(id, { vendor: g.vendor, sums: {}, counts: {} }) + const entry = acc.get(id) + if (g.vendor && !entry.vendor) entry.vendor = g.vendor + for (const f of numericFields) { + const v = g[f] + if (typeof v === 'number' && Number.isFinite(v)) { + entry.sums[f] = (entry.sums[f] ?? 0) + v + entry.counts[f] = (entry.counts[f] ?? 0) + 1 + } + } + } + } + const out: NodeMetricsGpu[] = [] + for (const [resourceId, entry] of acc) { + const gpu: NodeMetricsGpu = { resourceId, vendor: entry.vendor } + for (const f of numericFields) { + if (entry.counts[f] > 0) { + ;(gpu[f] as number) = entry.sums[f] / entry.counts[f] + } + } + out.push(gpu) + } + return out +} + +// Average total/inUse per env+resource across every sample that reported that pair. +function averageEnv(perSample: NodeMetricsEnvResource[][]): NodeMetricsEnvResource[] { + const acc = new Map< + string, + { env: string; resource: string; total: number; inUse: number; count: number } + >() + for (const list of perSample) { + for (const e of list ?? []) { + const key = `${e.env} ${e.resource}` + if (!acc.has(key)) { + acc.set(key, { env: e.env, resource: e.resource, total: 0, inUse: 0, count: 0 }) + } + const entry = acc.get(key) + entry.total += Number(e.total) || 0 + entry.inUse += Number(e.inUse) || 0 + entry.count += 1 + } + } + const out: NodeMetricsEnvResource[] = [] + for (const entry of acc.values()) { + out.push({ + env: entry.env, + resource: entry.resource, + total: entry.count > 0 ? entry.total / entry.count : 0, + inUse: entry.count > 0 ? entry.inUse / entry.count : 0 + }) + } + return out +} diff --git a/src/components/database/sql/PostgresClient.ts b/src/components/database/sql/PostgresClient.ts new file mode 100644 index 000000000..ee6e55a77 --- /dev/null +++ b/src/components/database/sql/PostgresClient.ts @@ -0,0 +1,155 @@ +import pg from 'pg' +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import { DATABASE_LOGGER } from '../../../utils/logging/common.js' +import { DB_CONNECTION_EVENTS, DB_EVENTS } from '../connectionEvents.js' +import { SqlClient, toSqlValue } from './SqlClient.js' + +const HEALTH_CHECK_INTERVAL_MS = 15_000 +const HEALTH_CHECK_TIMEOUT_MS = 5_000 +const CONNECT_TIMEOUT_MS = 10_000 +// the name of the lock that a node holds while it changes the schema +const SCHEMA_LOCK = 'ocean-node:schema' + +// the part of pg.Pool and pg.PoolClient that the client uses +type Queryable = Pick + +// PostgreSQL numbers its placeholders: `?` becomes `$1`, `$2`, ... Text in quotes stays. +export function numberPlaceholders(sql: string): string { + let count = 0 + return sql.replace(/'(?:[^']|'')*'|"[^"]*"|\?/g, (match) => + match === '?' ? `$${++count}` : match + ) +} + +// pg takes the user and the password from the connection string only, when it gets one. +// They are encoded: pg decodes them, so a "%41" in a password must not become an "A". +export function connectionString(config: OceanNodeDBConfig): string { + const url = new URL(config.url) + url.username = encodeURIComponent(config.username) + url.password = encodeURIComponent(config.password) + return url.toString() +} + +// what the SQL stores must write in a different way on PostgreSQL +const DIALECT = { + rowLock: ' FOR UPDATE', + // the default collation of a database follows a language + byteOrderText: 'TEXT COLLATE "C"' +} + +function statements(queryable: Queryable) { + const query = (sql: string, params: unknown[] = []) => + queryable.query(numberPlaceholders(sql), params.map(toSqlValue)) + return { + ...DIALECT, + exec: async (sql: string) => { + await queryable.query(sql) + }, + run: async (sql: string, params?: unknown[]) => ({ + changes: (await query(sql, params)).rowCount ?? 0 + }), + get: async (sql: string, params?: unknown[]) => (await query(sql, params)).rows[0], + all: async (sql: string, params?: unknown[]) => (await query(sql, params)).rows + } +} + +/** + * PostgreSQL through a connection pool. Several node instances can use one database: a + * transaction runs on one connection, and `rowLock` keeps a row for it. + */ +export class PostgresClient implements SqlClient { + readonly rowLock = DIALECT.rowLock + readonly byteOrderText = DIALECT.byteOrderText + private pool: pg.Pool + private healthTimer: NodeJS.Timeout + private connected = true + // runs each statement on a connection of the pool, outside a transaction + private pooled: Omit + + constructor(config: OceanNodeDBConfig) { + this.pool = new pg.Pool({ + connectionString: connectionString(config), + // a server that does not answer must give an error, so that the startup retries + connectionTimeoutMillis: CONNECT_TIMEOUT_MS, + // a BIGINT comes as text by default; the values of the node fit in a number + types: { + getTypeParser: (id: number, format?: any) => + id === pg.types.builtins.INT8 ? Number : pg.types.getTypeParser(id, format) + } + }) + // an idle connection that the server closes must not stop the process + this.pool.on('error', (error) => { + DATABASE_LOGGER.error(`PostgreSQL connection error: ${error.message}`) + }) + this.pooled = statements(this.pool) + this.healthTimer = setInterval(() => this.checkHealth(), HEALTH_CHECK_INTERVAL_MS) + this.healthTimer.unref() + } + + // Schema changes run one at a time: PostgreSQL fails one of two CREATE TABLE IF NOT + // EXISTS statements for the same table that run at the same time (two nodes that start + // together). The lock ends with the transaction. + exec(sql: string): Promise { + return this.transaction(async (tx) => { + await tx.get('SELECT pg_advisory_xact_lock(hashtext(?))', [SCHEMA_LOCK]) + await tx.exec(sql) + }) + } + + run(sql: string, params?: unknown[]): Promise<{ changes: number }> { + return this.pooled.run(sql, params) + } + + get(sql: string, params?: unknown[]): Promise { + return this.pooled.get(sql, params) + } + + all(sql: string, params?: unknown[]): Promise { + return this.pooled.all(sql, params) + } + + async transaction(work: (tx: SqlClient) => Promise): Promise { + const connection = await this.pool.connect() + const tx: SqlClient = { + ...statements(connection), + transaction: (nested) => nested(tx), + close: () => Promise.resolve() + } + try { + await connection.query('BEGIN') + const result = await work(tx) + await connection.query('COMMIT') + return result + } catch (error) { + await connection.query('ROLLBACK').catch(() => {}) + throw error + } finally { + connection.release() + } + } + + async close(): Promise { + clearInterval(this.healthTimer) + await this.pool.end() + } + + // tells the node when the database goes away and comes back, so that the indexer stops + // and starts again + private async checkHealth(): Promise { + // the timeout ends a check on a lost connection before the next check starts + const probe = { text: 'SELECT 1', query_timeout: HEALTH_CHECK_TIMEOUT_MS } + const alive = await this.pool.query(probe).then( + () => true, + () => false + ) + if (alive === this.connected) return + this.connected = alive + if (alive) { + DATABASE_LOGGER.info('PostgreSQL connection restored') + DB_CONNECTION_EVENTS.emit(DB_EVENTS.CONNECTION_RESTORED) + } else { + DATABASE_LOGGER.warn('PostgreSQL connection lost') + DB_CONNECTION_EVENTS.emit(DB_EVENTS.CONNECTION_LOST) + } + } +} diff --git a/src/components/database/sql/PostgresStores.ts b/src/components/database/sql/PostgresStores.ts new file mode 100644 index 000000000..f3b0df4d8 --- /dev/null +++ b/src/components/database/sql/PostgresStores.ts @@ -0,0 +1,18 @@ +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import type { DatabaseStores } from '../DatabaseFactory.js' +import { PostgresClient } from './PostgresClient.js' +import { buildSqlStores } from './SqlStores.js' + +// all the stores are in one database, on one connection pool +export async function createPostgresStores( + config: OceanNodeDBConfig +): Promise { + const client = new PostgresClient(config) + try { + return await buildSqlStores(config, () => client) + } catch (error) { + // the node tries again with a new client + await client.close() + throw error + } +} diff --git a/src/components/database/sql/SqlAccessList.ts b/src/components/database/sql/SqlAccessList.ts new file mode 100644 index 000000000..9eca78ade --- /dev/null +++ b/src/components/database/sql/SqlAccessList.ts @@ -0,0 +1,126 @@ +import type { AccessListUser } from '../../../@types/AccessList.js' +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import { DATABASE_LOGGER } from '../../../utils/logging/common.js' +import { AbstractAccessListDatabase, Condition } from '../BaseDatabase.js' +import type { SqlClient } from './SqlClient.js' +import { SqlDocuments } from './SqlDocuments.js' +import { orNull } from './orNull.js' + +// the largest number of access lists that a search by wallet gives +const SEARCH_HITS = 250 + +// the access list contracts of the indexed chains, each with its users +export class SqlAccessListDatabase extends AbstractAccessListDatabase { + private lists: SqlDocuments + + constructor(config: OceanNodeDBConfig, db: SqlClient) { + super(config) + this.lists = new SqlDocuments(db, 'access_list') + } + + init(): Promise { + return this.lists.init() + } + + // The deployment event. An AddressAdded event can come first and make the list: then + // only the deployment fields change, and the users stay. + create( + chainId: number, + contractAddress: string, + transferable: boolean, + block: number, + txId: string, + name?: string, + symbol?: string + ) { + const id = listId(chainId, contractAddress) + const deployment = { transferable, deploymentBlock: block, deploymentTxId: txId } + return orNull(`Error when upserting access list ${id}`, async () => { + await this.lists.modify(id, (list) => ({ + ...(list ?? newList(chainId, contractAddress)), + ...deployment, + ...(name != null && { name }), + ...(symbol != null && { symbol }) + })) + return { id } + }) + } + + retrieve(chainId: number, contractAddress: string) { + const id = listId(chainId, contractAddress) + return orNull(`Error when retrieving access list ${id}`, () => this.lists.get(id)) + } + + // a token id is in the list one time + addUser(chainId: number, contractAddress: string, user: AccessListUser) { + const id = listId(chainId, contractAddress) + const added: AccessListUser = { ...user, wallet: user.wallet.toLowerCase() } + const emptyList = { ...newList(chainId, contractAddress), transferable: false } + return orNull( + `Error when adding user ${added.wallet} to access list ${id}`, + async () => { + await this.lists.modify(id, (current) => { + const list = current ?? emptyList + const users: AccessListUser[] = list.users ?? [] + const exists = users.some((existing) => existing.tokenId === added.tokenId) + return exists ? null : { ...list, users: [...users, added] } + }) + return { id } + } + ) + } + + removeUserByTokenId(chainId: number, contractAddress: string, tokenId: number) { + const id = listId(chainId, contractAddress) + return orNull(`Error when removing tokenId ${tokenId} from ${id}`, async () => { + const list = await this.lists.modify( + id, + (current) => + current && { + ...current, + users: (current.users ?? []).filter( + (user: AccessListUser) => user.tokenId !== tokenId + ) + } + ) + if (!list) { + DATABASE_LOGGER.error( + `AddressRemoved on missing access list ${id} (tokenId=${tokenId}); ignoring.` + ) + } + return list ? { id } : null + }) + } + + async searchByWallet(wallet: string, chainId?: number): Promise { + const conditions: Condition[] = [{ paths: ['users.wallet'], anyOf: [wallet] }] + if (chainId !== undefined) conditions.push({ paths: ['chainId'], anyOf: [chainId] }) + const page = { offset: 0, limit: SEARCH_HITS } + const lists = await orNull( + `Error when searching access lists of ${wallet}`, + async () => { + return (await this.lists.find(conditions, [], page)).results + } + ) + return lists ?? [] + } + + delete(chainId: number, contractAddress: string) { + const id = listId(chainId, contractAddress) + return orNull(`Error when deleting access list ${id}`, async () => + (await this.lists.delete(id)) ? { id } : null + ) + } +} + +function newList(chainId: number, contractAddress: string) { + return { + chainId, + contractAddress: contractAddress.toLowerCase(), + users: [] as AccessListUser[] + } +} + +function listId(chainId: number, contractAddress: string): string { + return `${chainId}-${contractAddress.toLowerCase()}` +} diff --git a/src/components/database/sql/SqlAuthToken.ts b/src/components/database/sql/SqlAuthToken.ts new file mode 100644 index 000000000..46cc8fce0 --- /dev/null +++ b/src/components/database/sql/SqlAuthToken.ts @@ -0,0 +1,90 @@ +import { DATABASE_LOGGER } from '../../../utils/logging/common.js' +import { + type AuthToken, + type AuthTokenDatabaseProvider, + toUnixMs +} from '../AuthTokenDatabase.js' +import type { SqlClient } from './SqlClient.js' + +type TokenRow = { + address: string + createdAt: number + // a database of an older release can hold a date text + validUntil: number | string | null + isValid: number + chainId: string | null +} + +// one row per token: "token" = the sha256 of the token, never the token itself +export class SqlAuthToken implements AuthTokenDatabaseProvider { + constructor(private db: SqlClient) {} + + init(): Promise { + return this.db.exec(` + CREATE TABLE IF NOT EXISTS "authTokens" ( + "token" TEXT PRIMARY KEY, + "address" TEXT NOT NULL, + "createdAt" BIGINT, + "validUntil" BIGINT, + "isValid" BIGINT DEFAULT 1, + "chainId" TEXT + )`) + } + + async createToken( + tokenId: string, + address: string, + createdAt: number, + validUntil: number | null = null, + chainId?: string | null + ): Promise { + await this.db.run( + `INSERT INTO "authTokens" ("token", "address", "createdAt", "validUntil", "isValid", "chainId") + VALUES (?, ?, ?, ?, 1, ?) + ON CONFLICT ("token") DO UPDATE SET + "address" = excluded."address", "createdAt" = excluded."createdAt", + "validUntil" = excluded."validUntil", "isValid" = 1, "chainId" = excluded."chainId"`, + // text: SQLite stores the number 8996 in a TEXT column as '8996.0' + [tokenId, address, createdAt, validUntil, chainId == null ? null : String(chainId)] + ) + } + + // null for an unknown, invalidated or expired token; an expired one is invalidated + async validateTokenEntry(tokenId: string): Promise { + const row = await this.db.get( + 'SELECT * FROM "authTokens" WHERE "token" = ?', + [tokenId] + ) + if (!row || !row.isValid) { + return null + } + const validUntil = row.validUntil === null ? null : toUnixMs(row.validUntil) + // an end that is not a time counts as expired + if (validUntil !== null && !(validUntil >= Date.now())) { + DATABASE_LOGGER.debug('Expired auth token invalidated') + await this.invalidateTokenEntry(tokenId) + return null + } + return { + address: row.address, + createdAt: Number(row.createdAt), + validUntil, + isValid: true, + chainId: row.chainId ?? null + } + } + + async invalidateTokenEntry(tokenId: string): Promise { + await this.db.run('UPDATE "authTokens" SET "isValid" = 0 WHERE "token" = ?', [ + tokenId + ]) + } + + async deleteExpiredTokens(now: number): Promise { + const { changes } = await this.db.run( + 'DELETE FROM "authTokens" WHERE "validUntil" < ? OR "isValid" = 0', + [now] + ) + return changes + } +} diff --git a/src/components/database/sql/SqlBucketRegistry.ts b/src/components/database/sql/SqlBucketRegistry.ts new file mode 100644 index 000000000..5990a4377 --- /dev/null +++ b/src/components/database/sql/SqlBucketRegistry.ts @@ -0,0 +1,71 @@ +import type { BucketRegistry, BucketRow } from '../BucketRegistry.js' +import type { SqlClient } from './SqlClient.js' + +const COLUMNS = '"bucketId", "owner", "accessListJson", "createdAt", "label"' + +// one row per persistent-storage bucket +export class SqlBucketRegistry implements BucketRegistry { + constructor(private db: SqlClient) {} + + init(): Promise { + return this.db.exec(` + CREATE TABLE IF NOT EXISTS persistent_storage_buckets ( + "bucketId" TEXT PRIMARY KEY, + "owner" TEXT NOT NULL, + "accessListJson" TEXT NOT NULL, + "createdAt" BIGINT NOT NULL, + "label" TEXT + )`) + } + + // a conflict changes the access list only, so a second create never removes a new name + async upsertBucket( + bucketId: string, + owner: string, + accessListJson: string, + createdAt: number, + label: string | null + ): Promise { + await this.db.run( + `INSERT INTO persistent_storage_buckets (${COLUMNS}) VALUES (?, ?, ?, ?, ?) + ON CONFLICT ("bucketId") DO UPDATE SET "accessListJson" = excluded."accessListJson"`, + [bucketId, owner, accessListJson, createdAt, label] + ) + } + + async getBucket(bucketId: string): Promise { + const row = await this.db.get( + `SELECT ${COLUMNS} FROM persistent_storage_buckets WHERE "bucketId" = ?`, + [bucketId] + ) + return row ?? null + } + + listBucketsByOwner(owner: string): Promise { + return this.db.all( + `SELECT ${COLUMNS} FROM persistent_storage_buckets + WHERE "owner" = ? ORDER BY "createdAt" ASC`, + [owner] + ) + } + + async deleteBucket(bucketId: string): Promise { + const { changes } = await this.db.run( + 'DELETE FROM persistent_storage_buckets WHERE "bucketId" = ?', + [bucketId] + ) + return changes === 1 + } + + async updateBucketLabel( + bucketId: string, + owner: string, + label: string | null + ): Promise { + const { changes } = await this.db.run( + 'UPDATE persistent_storage_buckets SET "label" = ? WHERE "bucketId" = ? AND "owner" = ?', + [label, bucketId, owner] + ) + return changes === 1 + } +} diff --git a/src/components/database/sql/SqlClient.ts b/src/components/database/sql/SqlClient.ts new file mode 100644 index 000000000..816400851 --- /dev/null +++ b/src/components/database/sql/SqlClient.ts @@ -0,0 +1,69 @@ +// The SQL stores talk to SQLite and PostgreSQL through this interface, so each store is +// written one time. A statement is valid on the two databases when it follows these rules: +// - placeholders are `?` +// - identifiers are in double quotes (PostgreSQL changes an unquoted name to lower case) +// - column types are TEXT, BIGINT and DOUBLE PRECISION; a boolean is 0 or 1, JSON is TEXT +// - no JSON function, no database-specific function +export interface SqlClient { + // Put after a SELECT in a transaction that reads a row and then writes it: the row + // stays locked until the transaction ends. Empty when the database locks without it. + readonly rowLock: string + // The type of a text column that is compared and sorted by its bytes, so that a text + // search gives the same result and the same order on all databases. + readonly byteOrderText: string + // one or more statements with no parameters (schema changes) + exec(sql: string): Promise + run(sql: string, params?: unknown[]): Promise<{ changes: number }> + get(sql: string, params?: unknown[]): Promise + all(sql: string, params?: unknown[]): Promise + // `work` gets a client that runs inside the transaction; a throw rolls back + transaction(work: (tx: SqlClient) => Promise): Promise + close(): Promise +} + +// The value of a parameter: the drivers refuse `undefined`, and a boolean is 0 or 1. +export function toSqlValue(value: unknown): unknown { + if (value === undefined) return null + if (typeof value === 'boolean') return value ? 1 : 0 + return value +} + +// `?, ?, ?` for a list of values +export function placeholders(count: number): string { + return Array(count).fill('?').join(', ') +} + +export function insertRow( + db: SqlClient, + table: string, + row: Record +): Promise<{ changes: number }> { + const columns = Object.keys(row) + return db.run( + `INSERT INTO ${table} (${columns.map((column) => `"${column}"`).join(', ')}) + VALUES (${placeholders(columns.length)})`, + Object.values(row) + ) +} + +// The WHERE part of a statement, built from optional filters. All conditions must hold. +export class Conditions { + readonly params: unknown[] = [] + private parts: string[] = [] + + add(condition: string, ...params: unknown[]): this { + this.parts.push(condition) + this.params.push(...params) + return this + } + + // `"column" IN (...)`; no condition for an absent or empty list + addIn(expression: string, values?: readonly unknown[]): this { + if (!values || values.length === 0) return this + return this.add(`${expression} IN (${placeholders(values.length)})`, ...values) + } + + toString(): string { + return this.parts.length > 0 ? ` WHERE ${this.parts.join(' AND ')}` : '' + } +} diff --git a/src/components/database/sql/SqlCompute.ts b/src/components/database/sql/SqlCompute.ts new file mode 100644 index 000000000..c01b0950e --- /dev/null +++ b/src/components/database/sql/SqlCompute.ts @@ -0,0 +1,449 @@ +import { + C2DStatusNumber, + C2DStatusText, + type ContainerMetricsSnapshot, + type DBComputeJob +} from '../../../@types/C2D/C2D.js' +import { + ServiceStatusNumber, + SERVICE_START_PENDING_STATUSES, + type ServiceJob, + type ServiceModelDownload, + type ServiceReadiness +} from '../../../@types/C2D/ServiceOnDemand.js' +import { DATABASE_LOGGER } from '../../../utils/logging/common.js' +import type { ComputeDatabaseProvider, ServiceJobExpected } from '../C2DDatabase.js' +import { + ACTIVE_SERVICE_STATUSES, + EXPIRABLE_SERVICE_STATUSES, + agreementIdVariants, + holdsReservation, + jobToBody, + matchesExpected, + oldImageCutoff, + withJobIdHash +} from '../computeJobs.js' +import { Conditions, SqlClient, insertRow } from './SqlClient.js' + +const JOBS = 'c2djobs' +const SERVICE_JOBS = 'service_jobs' +const IMAGES = 'docker_images' +const LOCKS = 'service_locks' +// the input DIDs of a job are one string in the "inputDID" column +const DID_SEPARATOR = '__,__' + +// "dateCreated" and "dateFinished" of a job are decimal seconds in a TEXT column: as text, +// '999999999.5' is above '1785760660', so they are compared and sorted as numbers +const asNumber = (column: string) => `CAST("${column}" AS DOUBLE PRECISION)` +const NEWEST_CREATED_FIRST = ` ORDER BY ${asNumber('dateCreated')} DESC` + +// PostgreSQL fails a query that casts a text that is not a number, for all rows: such a +// date must not get into the table +function seconds(date: string | null | undefined): string | null { + if (date === null || date === undefined || date === '') return null + if (!Number.isFinite(Number(date))) { + throw new Error(`The date of a job must be in Unix seconds, not '${date}'`) + } + return date +} + +// a body is JSON text; a database from an older release holds it as bytes +export function parseBody(body: string | Uint8Array): any { + return JSON.parse(typeof body === 'string' ? body : Buffer.from(body).toString()) +} + +function rowToJob(row: any): DBComputeJob { + const { body, expireTimestamp, inputDID, ...columns } = row + return withJobIdHash({ + ...columns, + inputDID: typeof inputDID === 'string' ? splitDids(inputDID) : inputDID, + ...parseBody(body), + maxJobDuration: expireTimestamp + }) +} + +function splitDids(dids: string): string[] { + return dids === '' ? [] : dids.split(DID_SEPARATOR) +} + +// the columns of a service job that change with it; the body holds the full job +function serviceJobColumns(job: ServiceJob): Record { + return { + owner: job.owner, + clusterHash: job.clusterHash, + status: job.status, + expiresAt: job.expiresAt, + outputArchiveCount: (job.outputArchives ?? []).length, + body: JSON.stringify({ ...job, updatedAt: Date.now() }) + } +} + +const rowToServiceJob = (row: { body: string | Uint8Array }): ServiceJob => + parseBody(row.body) + +// an owner matches in any letter case +function ownerIs(owner: string): [string, string] { + return ['LOWER("owner") = ?', owner.toLowerCase()] +} + +export class SqlCompute implements ComputeDatabaseProvider { + constructor(private db: SqlClient) {} + + init(): Promise { + // "expireTimestamp" holds the maximum job duration + return this.db.exec(` + CREATE TABLE IF NOT EXISTS ${JOBS} ( + "jobId" TEXT PRIMARY KEY, "owner" TEXT, "did" TEXT, "dateCreated" TEXT, + "dateFinished" TEXT, "status" BIGINT, "statusText" TEXT, "inputDID" TEXT, + "algoDID" TEXT, "agreementId" TEXT, "expireTimestamp" BIGINT, "environment" TEXT, + "body" TEXT + ); + CREATE TABLE IF NOT EXISTS ${IMAGES} ( + "image" TEXT PRIMARY KEY, "lastUsedTimestamp" BIGINT NOT NULL + ); + CREATE TABLE IF NOT EXISTS ${SERVICE_JOBS} ( + "serviceId" TEXT PRIMARY KEY, "owner" TEXT, "clusterHash" TEXT, "status" BIGINT, + "expiresAt" BIGINT, "dateCreated" TEXT, "outputArchiveCount" BIGINT, "body" TEXT + ); + CREATE TABLE IF NOT EXISTS ${LOCKS} ( + "serviceId" TEXT PRIMARY KEY, "holder" TEXT NOT NULL, "acquiredAt" BIGINT NOT NULL + );`) + } + + // ── C2D jobs ──────────────────────────────────────────────────────── + + async newJob(job: DBComputeJob): Promise { + await insertRow(this.db, JOBS, { + jobId: job.jobId, + owner: job.owner, + did: job.did, + dateCreated: seconds(job.dateCreated) ?? String(Date.now() / 1000), + dateFinished: null, + status: job.status || C2DStatusNumber.JobStarted, + statusText: job.statusText || C2DStatusText.JobStarted, + inputDID: job.inputDID?.join(DID_SEPARATOR) ?? null, + algoDID: job.algoDID, + agreementId: job.agreementId, + expireTimestamp: job.maxJobDuration, + environment: job.environment, + body: JSON.stringify(jobToBody(job)) + }) + DATABASE_LOGGER.debug( + `C2D job ${job.clusterHash}-${job.jobId} created (env ${job.environment})` + ) + return job.jobId + } + + async getJob( + jobId?: string, + agreementId?: string, + owner?: string + ): Promise { + const where = new Conditions() + if (jobId) where.add('"jobId" = ?', jobId) + if (agreementId) where.addIn('"agreementId"', agreementIdVariants(agreementId)) + if (owner) where.add(...ownerIs(owner)) + const jobs = await this.selectJobs(where) + if (jobs.length === 0) { + DATABASE_LOGGER.debug( + `Could not find any job with jobId: ${jobId}, agreementId: ${agreementId}, or owner: ${owner} in database!` + ) + } + return jobs + } + + async updateJob(job: DBComputeJob): Promise { + const { changes } = await this.db.run( + `UPDATE ${JOBS} + SET "owner" = ?, "status" = ?, "statusText" = ?, "expireTimestamp" = ?, "body" = ?, + "dateFinished" = ? + WHERE "jobId" = ?`, + [ + job.owner, + job.status, + job.statusText, + job.maxJobDuration, + JSON.stringify(jobToBody(job)), + seconds(job.dateFinished), + job.jobId + ] + ) + return changes + } + + async getRunningJobs(engine?: string, environment?: string): Promise { + const where = new Conditions().add('"dateFinished" IS NULL') + if (environment) where.add('"environment" = ?', environment) + const jobs = await this.selectJobs(where, ` ORDER BY ${asNumber('dateCreated')} ASC`) + // the engine (clusterHash) is in the body, so it is filtered here + return engine ? jobs.filter((job) => job.clusterHash === engine) : jobs + } + + async deleteJob(jobId: string): Promise { + const { changes } = await this.db.run(`DELETE FROM ${JOBS} WHERE "jobId" = ?`, [ + jobId + ]) + return changes === 1 + } + + getFinishedJobs(environments?: string[]): Promise { + return this.selectJobs( + new Conditions() + .add('"dateFinished" IS NOT NULL') + .addIn('"environment"', environments), + ` ORDER BY ${asNumber('dateFinished')} DESC` + ) + } + + /** @param fromTimestamp lower bound in Unix seconds — see C2DDatabase.getJobs. */ + getJobs( + environments?: string[], + fromTimestamp?: number, + consumerAddrs?: string[], + status?: C2DStatusNumber, + runningJobs?: boolean + ): Promise { + const where = new Conditions().addIn('"environment"', environments) + const hasFrom = fromTimestamp !== undefined && fromTimestamp !== null + if (runningJobs) { + where.add('"status" = ?', C2DStatusNumber.RunningAlgorithm) + if (hasFrom) where.add(`${asNumber('dateCreated')} >= ?`, fromTimestamp) + } else { + if (hasFrom) where.add(`${asNumber('dateFinished')} >= ?`, fromTimestamp) + // C2DStatusNumber.JobStarted is 0: a truthiness check would drop the filter + if (status !== undefined && status !== null) where.add('"status" = ?', status) + } + where.addIn( + 'LOWER("owner")', + consumerAddrs?.map((address) => address.toLowerCase()) + ) + return this.selectJobs(where, NEWEST_CREATED_FIRST) + } + + getJobsByStatus( + environments: string[], + status: C2DStatusNumber[] + ): Promise { + return this.selectJobs( + new Conditions().addIn('"environment"', environments).addIn('"status"', status), + NEWEST_CREATED_FIRST + ) + } + + private async selectJobs(where: Conditions, orderBy = ''): Promise { + const rows = await this.db.all( + `SELECT * FROM ${JOBS}${where}${orderBy}`, + where.params + ) + return rows.map(rowToJob) + } + + // ── Docker images ─────────────────────────────────────────────────── + + async updateImage(image: string): Promise { + await this.db.run( + `INSERT INTO ${IMAGES} ("image", "lastUsedTimestamp") VALUES (?, ?) + ON CONFLICT ("image") DO UPDATE SET "lastUsedTimestamp" = excluded."lastUsedTimestamp"`, + [image, Math.floor(Date.now() / 1000)] + ) + DATABASE_LOGGER.debug(`Updated image usage timestamp for ${image}`) + } + + async deleteImage(image: string): Promise { + await this.db.run(`DELETE FROM ${IMAGES} WHERE "image" = ?`, [image]) + DATABASE_LOGGER.debug(`Deleted image ${image}`) + } + + async getOldImages(retentionDays: number): Promise { + const cutoff = oldImageCutoff(retentionDays) + const rows = await this.db.all<{ image: string }>( + `SELECT "image" FROM ${IMAGES} + WHERE "lastUsedTimestamp" < ? ORDER BY "lastUsedTimestamp" ASC`, + [cutoff] + ) + return rows.map((row) => row.image) + } + + // ── Service-on-Demand jobs ────────────────────────────────────────── + + async newServiceJob(job: ServiceJob): Promise { + await insertRow(this.db, SERVICE_JOBS, { + serviceId: job.serviceId, + dateCreated: job.dateCreated, + ...serviceJobColumns(job) + }) + } + + getServiceJob(serviceId?: string, owner?: string): Promise { + const where = new Conditions() + if (serviceId) where.add('"serviceId" = ?', serviceId) + if (owner) where.add(...ownerIs(owner)) + return this.selectServiceJobs(where) + } + + async updateServiceJob(job: ServiceJob): Promise { + const columns = serviceJobColumns(job) + const assignments = Object.keys(columns).map((column) => `"${column}" = ?`) + const { changes } = await this.db.run( + `UPDATE ${SERVICE_JOBS} SET ${assignments.join(', ')} WHERE "serviceId" = ?`, + [...Object.values(columns), job.serviceId] + ) + return changes + } + + updateServiceJobMetrics( + serviceId: string, + expected: ServiceJobExpected, + runtimeMetrics: ContainerMetricsSnapshot + ): Promise { + return this.patchServiceJobBody(serviceId, expected, 'metrics', (body) => { + body.runtimeMetrics = runtimeMetrics + }) + } + + // `readiness` is undefined for a service that the node cannot probe + updateServiceJobReadiness( + serviceId: string, + expected: ServiceJobExpected, + readiness: ServiceReadiness | undefined, + modelDownload?: ServiceModelDownload + ): Promise { + return this.patchServiceJobBody(serviceId, expected, 'readiness', (body) => { + if (readiness) body.readiness = readiness + // changed only by a new sample: the last figures stay as the record of the download + if (modelDownload) body.modelDownload = modelDownload + }) + } + + // Every status before Expired holds the paid reservation; see ACTIVE_SERVICE_STATUSES. + async getRunningServiceJobs(clusterHash?: string): Promise { + const jobs = await this.selectServiceJobs( + inCluster(clusterHash).addIn('"status"', ACTIVE_SERVICE_STATUSES) + ) + // filtered here: the payment is in the body + return jobs.filter(holdsReservation) + } + + // The services that the sweep must release: past their end, in a status that still + // holds the reservation. + getExpiredServiceJobs(clusterHash?: string): Promise { + return this.selectServiceJobs( + inCluster(clusterHash) + .addIn('"status"', EXPIRABLE_SERVICE_STATUSES) + .add('"expiresAt" <= ?', Date.now()) + ) + } + + // Expired services that still have output archives. Expired rows stay forever, so the + // archive filter is in SQL: a cleanup run must not read every service that ever expired. + getExpiredServiceJobsBefore( + expiresBefore: number, + clusterHash?: string + ): Promise { + return this.selectServiceJobs( + inCluster(clusterHash) + .add('"status" = ?', ServiceStatusNumber.Expired) + .add('"expiresAt" <= ?', expiresBefore) + .add('"outputArchiveCount" > 0') + ) + } + + // the services that are in the middle of a start and need the background loop + getPendingServiceStarts(clusterHash?: string): Promise { + return this.selectServiceJobs( + inCluster(clusterHash).addIn('"status"', SERVICE_START_PENDING_STATUSES) + ) + } + + private async selectServiceJobs(where: Conditions): Promise { + const rows = await this.db.all( + `SELECT "body" FROM ${SERVICE_JOBS}${where}`, + where.params + ) + return rows.map(rowToServiceJob) + } + + // Writes ONE best-effort field (metrics, readiness) of a service job and no lifecycle + // field. In one transaction it reads the row again, checks that owner, cluster, status + // and container did not change, applies `patch` to the current body and writes the body + // only. A mismatch or an error gives false: the caller samples again later. + private async patchServiceJobBody( + serviceId: string, + expected: ServiceJobExpected, + what: string, + patch: (body: ServiceJob) => void + ): Promise { + try { + return await this.db.transaction(async (tx) => { + const row = await tx.get( + `SELECT "body" FROM ${SERVICE_JOBS} WHERE "serviceId" = ?${tx.rowLock}`, + [serviceId] + ) + const body = row && rowToServiceJob(row) + if (!body || !matchesExpected(body, expected)) return false + patch(body) + const { changes } = await tx.run( + `UPDATE ${SERVICE_JOBS} SET "body" = ? WHERE "serviceId" = ? AND "status" = ?`, + [JSON.stringify({ ...body, updatedAt: Date.now() }), serviceId, expected.status] + ) + return changes > 0 + }) + } catch (error) { + DATABASE_LOGGER.error( + `Error while updating ${what} of service job ${serviceId}: ${error.message}` + ) + return false + } + } + + // ── Service locks ─────────────────────────────────────────────────── + // A row is an exclusive start, stop or restart of a service, held by `holder` (one id + // per process). The holder refreshes "acquiredAt" while it works; a row older than the + // stale limit is from a holder that crashed, and another holder can take it. + + // One statement inserts the row or takes a stale one, so two holders that race can + // never both get the lock. + async acquireServiceLock( + serviceId: string, + holder: string, + staleMs: number + ): Promise { + const now = Date.now() + const { changes } = await this.db.run( + `INSERT INTO ${LOCKS} ("serviceId", "holder", "acquiredAt") VALUES (?, ?, ?) + ON CONFLICT ("serviceId") DO UPDATE + SET "holder" = excluded."holder", "acquiredAt" = excluded."acquiredAt" + WHERE ${LOCKS}."acquiredAt" <= ?`, + [serviceId, holder, now, now - staleMs] + ) + return changes === 1 + } + + // only the holder releases: a stale lock that another holder took stays + async releaseServiceLock(serviceId: string, holder: string): Promise { + await this.db.run(`DELETE FROM ${LOCKS} WHERE "serviceId" = ? AND "holder" = ?`, [ + serviceId, + holder + ]) + } + + async refreshServiceLocks(holder: string): Promise { + await this.db.run(`UPDATE ${LOCKS} SET "acquiredAt" = ? WHERE "holder" = ?`, [ + Date.now(), + holder + ]) + } + + async isServiceLocked(serviceId: string, staleMs: number): Promise { + const row = await this.db.get<{ acquiredAt: number }>( + `SELECT "acquiredAt" FROM ${LOCKS} WHERE "serviceId" = ?`, + [serviceId] + ) + return !!row && row.acquiredAt > Date.now() - staleMs + } +} + +function inCluster(clusterHash?: string): Conditions { + const where = new Conditions() + return clusterHash ? where.add('"clusterHash" = ?', clusterHash) : where +} diff --git a/src/components/database/sql/SqlConfig.ts b/src/components/database/sql/SqlConfig.ts new file mode 100644 index 000000000..572f63cb5 --- /dev/null +++ b/src/components/database/sql/SqlConfig.ts @@ -0,0 +1,35 @@ +import type { ConfigDatabase } from '../ConfigDatabase.js' +import { orNull } from './orNull.js' +import type { SqlClient } from './SqlClient.js' + +// node-wide key/value settings; a database error gives null, as the interface says +export class SqlConfigDatabase implements ConfigDatabase { + constructor(private db: SqlClient) {} + + init(): Promise { + return this.db.exec( + 'CREATE TABLE IF NOT EXISTS config ("key" TEXT NOT NULL PRIMARY KEY, "value" TEXT)' + ) + } + + createOrUpdateConfig(key: string, value: string) { + return orNull(`Error when writing config value ${key}`, async () => { + await this.db.run( + `INSERT INTO config ("key", "value") VALUES (?, ?) + ON CONFLICT ("key") DO UPDATE SET "value" = excluded."value"`, + [key, String(value)] + ) + return { key, value } + }) + } + + retrieveValue(key: string) { + return orNull(`Error when reading config value ${key}`, async () => { + const row = await this.db.get<{ value: string }>( + 'SELECT "value" FROM config WHERE "key" = ?', + [key] + ) + return { value: row?.value ?? null } + }) + } +} diff --git a/src/components/database/sql/SqlDdo.ts b/src/components/database/sql/SqlDdo.ts new file mode 100644 index 000000000..0542efa6b --- /dev/null +++ b/src/components/database/sql/SqlDdo.ts @@ -0,0 +1,82 @@ +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import { validateDDO } from '../../../utils/asset.js' +import { DATABASE_LOGGER } from '../../../utils/logging/common.js' +import { + AbstractDdoDatabase, + Condition, + DdoSchema, + Found, + Page, + SortSpec, + TagCount +} from '../BaseDatabase.js' +import { DDO_FIELDS } from '../assetQuery.js' +import { DDO_SCHEMA_NAMES, ddoSchemaName } from '../ddoSchemas.js' +import type { SqlClient } from './SqlClient.js' +import { SqlDocuments } from './SqlDocuments.js' +import { orNull } from './orNull.js' + +// the DDOs of the indexed chains: id = the DID +export class SqlDdoDatabase extends AbstractDdoDatabase { + private ddos: SqlDocuments + + constructor(config: OceanNodeDBConfig, db: SqlClient) { + super(config) + this.ddos = new SqlDocuments(db, 'ddo') + } + + init(): Promise { + return this.ddos.init() + } + + getSchemas(): DdoSchema[] { + return DDO_SCHEMA_NAMES.map((index) => ({ index })) + } + + find(conditions: Condition[], sort: SortSpec[], page: Page): Promise { + return this.ddos.find(conditions, sort, page) + } + + // the field table keeps text in lower case, so the tags are in lower case + async tags(conditions: Condition[], limit: number): Promise { + const counts = await this.ddos.valueCounts(DDO_FIELDS.tags, conditions, limit) + return counts.map(({ value, count }) => ({ tag: value, count })) + } + + create(ddo: Record): Promise { + const schema = ddoSchemaName(ddo) + if (!DDO_SCHEMA_NAMES.includes(schema)) { + throw new Error(`Schema for version ${ddo.version} not found`) + } + return orNull(`Error when saving DDO entry ${ddo.id}`, async () => { + // the nft is in indexedMetadata; a second copy fails the validation + if (ddo?.indexedMetadata?.nft) delete ddo.nft + if ((await validateDDO(ddo)) !== true) { + throw new Error(`Validation of DDO with schema version ${ddo.version} failed`) + } + await this.ddos.put(ddo.id, ddo) + return ddo + }) + } + + async retrieve(id: string): Promise { + const ddo = await this.ddos.get(id) + if (!ddo) { + DATABASE_LOGGER.debug(`DDO entry with ID ${id} not found.`) + } + return ddo + } + + // a DDO has one row, so an update that changes the schema needs no cleanup + update(ddo: Record): Promise { + return this.create(ddo) + } + + async delete(id: string): Promise { + return (await this.ddos.delete(id)) ? { id } : undefined + } + + deleteAllAssetsFromChain(chainId: number): Promise { + return this.ddos.deleteMatching([{ paths: DDO_FIELDS.chainId, anyOf: [chainId] }]) + } +} diff --git a/src/components/database/sql/SqlDdoState.ts b/src/components/database/sql/SqlDdoState.ts new file mode 100644 index 000000000..01f7ef813 --- /dev/null +++ b/src/components/database/sql/SqlDdoState.ts @@ -0,0 +1,69 @@ +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import { AbstractDdoStateDatabase, DdoStateKey } from '../BaseDatabase.js' +import type { SqlClient } from './SqlClient.js' +import { SqlDocuments } from './SqlDocuments.js' +import { orNull } from './orNull.js' + +// the largest number of states that a search gives, as on Elasticsearch +const SEARCH_HITS = 10 + +// the result of the indexing of each DDO: id = the DID +export class SqlDdoStateDatabase extends AbstractDdoStateDatabase { + private states: SqlDocuments + + constructor(config: OceanNodeDBConfig, db: SqlClient) { + super(config) + this.states = new SqlDocuments(db, 'ddo_state') + } + + init(): Promise { + return this.states.init() + } + + create( + chainId: number, + did: string, + nftAddress: string, + txId: string = ' ', + valid: boolean = true, + errorMsg: string = ' ' + ) { + const state = { chainId, nft: nftAddress, txId, valid, error: errorMsg } + return orNull(`Error when saving the ddo state of ${did}`, async () => { + await this.states.put(did, { did, ...state }) + return { id: did, ...state } + }) + } + + retrieve(did: string) { + return orNull(`Error when retrieving the ddo state of ${did}`, () => + this.states.get(did) + ) + } + + find(key: DdoStateKey) { + const [field, value] = Object.entries(key)[0] + return orNull('Error when searching the ddo states', async () => { + const page = { offset: 0, limit: SEARCH_HITS } + return (await this.states.find([{ paths: [field], anyOf: [value] }], [], page)) + .results + }) + } + + update( + chainId: number, + did: string, + nftAddress: string, + txId?: string, + valid?: boolean, + errorMsg?: string + ) { + return this.create(chainId, did, nftAddress, txId, valid, errorMsg) + } + + delete(did: string) { + return orNull(`Error when deleting the ddo state of ${did}`, async () => + (await this.states.delete(did)) ? { id: did } : null + ) + } +} diff --git a/src/components/database/sql/SqlDocuments.ts b/src/components/database/sql/SqlDocuments.ts new file mode 100644 index 000000000..b8d6d6af1 --- /dev/null +++ b/src/components/database/sql/SqlDocuments.ts @@ -0,0 +1,358 @@ +import type { Condition, Found, Page, Scalar, SortSpec } from '../BaseDatabase.js' +import { SqlClient, placeholders } from './SqlClient.js' + +type Document = Record + +interface FieldRow { + path: string + str: string + num: number | null +} + +interface SqlPart { + sql: string + params: unknown[] +} + +// A longer path gets no field row: PostgreSQL limits the size of an index entry, and a +// path of this length is a free-form key, not a field that a search names. +const PATH_LIMIT = 128 +// A field row keeps the full text, but the index has only its first KEY_LIMIT characters: +// PostgreSQL limits the size of an index entry. A comparison uses the full text. +const KEY_LIMIT = 512 +const KEY = `substr("str", 1, ${KEY_LIMIT})` +// rows or ids for one statement; the databases limit the number of parameters +const CHUNK_SIZE = 200 +const MATCH_ALL: SqlPart = { sql: '1 = 1', params: [] } +const MATCH_NONE: SqlPart = { sql: '1 = 0', params: [] } +const NUMBER_TEXT = /^-?\d+(\.\d+)?$/ +const RANGE_OPERATORS = { gt: '>', gte: '>=', lt: '<', lte: '<=' } + +// the value as the field table keeps it: lower case, so a comparison ignores the case +export function searchText(value: unknown): string { + // PostgreSQL refuses U+0000 in a text, and a DDO would not be stored + return String(value).toLowerCase().replaceAll('\u0000', '') +} + +// the indexed part of a text; SQL substr counts code points, as the spread does +function keyOf(text: string): string { + return [...text].slice(0, KEY_LIMIT).join('') +} + +// the number that a value holds: a number, or a text such as '10' or '2.5' +export function numberOf(value: unknown): number | null { + if (typeof value === 'number') return Number.isFinite(value) ? value : null + return typeof value === 'string' && NUMBER_TEXT.test(value) ? Number(value) : null +} + +// One row for each leaf value of the document. The path has no array index: the path of +// the type of each service is "services.type". +export function flattenDocument( + value: unknown, + path: string = '', + rows: FieldRow[] = [] +): FieldRow[] { + if (value === null || value === undefined) return rows + if (Array.isArray(value)) { + value.forEach((item) => flattenDocument(item, path, rows)) + } else if (typeof value === 'object') { + for (const [key, child] of Object.entries(value)) { + // PostgreSQL refuses U+0000 in a text, and the path is a text + const name = key.replaceAll('\u0000', '') + flattenDocument(child, path ? `${path}.${name}` : name, rows) + } + } else if (path.length <= PATH_LIMIT) { + rows.push({ path, str: searchText(value), num: numberOf(value) }) + } + return rows +} + +/** + * JSON documents in a SQL table, with a field table that makes them searchable with + * conditions (see Condition in BaseDatabase.ts). Tables: `` ("id", "doc") and + * `
_fields` (one row for each leaf value of each document: its path, its value as + * lower-case text, and its value as a number when it is one). A path has no array index, + * so a field in an array matches when one element matches. + */ +export class SqlDocuments { + private fields: string + + constructor( + private db: SqlClient, + private table: string + ) { + this.fields = `${table}_fields` + } + + init(): Promise { + const { fields } = this + return this.db.exec(` + CREATE TABLE IF NOT EXISTS ${this.table} ( + "id" ${this.db.byteOrderText} PRIMARY KEY, "doc" TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS ${fields} ( + "id" ${this.db.byteOrderText} NOT NULL, "path" TEXT NOT NULL, + "str" ${this.db.byteOrderText}, "num" DOUBLE PRECISION + ); + CREATE INDEX IF NOT EXISTS ${fields}_id ON ${fields} ("id"); + CREATE INDEX IF NOT EXISTS ${fields}_key ON ${fields} ("path", ${KEY}); + CREATE INDEX IF NOT EXISTS ${fields}_num ON ${fields} ("path", "num");`) + } + + // `extraFields` are searchable but are not part of the document + put(id: string, doc: Document, extraFields: Document = {}): Promise { + return this.db.transaction((tx) => this.write(tx, id, doc, extraFields)) + } + + get(id: string): Promise { + return this.read(this.db, id) + } + + // Reads the document (null when absent), applies `change` and writes the result, in one + // transaction. `change` returns null to keep the document as it is. + modify( + id: string, + change: (doc: Document | null) => Document | null + ): Promise { + return this.db.transaction(async (tx) => { + // Only a row that exists can be locked. Without it, two callers that make the same + // document at the same time both read "absent", and one change is lost. The JSON + // null is the absent document. + await tx.run( + `INSERT INTO ${this.table} ("id", "doc") VALUES (?, 'null') + ON CONFLICT ("id") DO NOTHING`, + [id] + ) + const current = await this.read(tx, id, tx.rowLock) + const changed = change(current) + if (changed) await this.write(tx, id, changed, {}) + else if (!current) await this.remove(tx, id) + return changed + }) + } + + delete(id: string): Promise { + return this.db.transaction((tx) => this.remove(tx, id)) + } + + // the documents that meet all conditions, sorted by `sort` and then by id + async find(conditions: Condition[], sort: SortSpec[], page: Page): Promise { + const where = this.where(conditions) + const total = await this.db.get<{ count: number }>( + `SELECT COUNT(*) AS "count" FROM ${this.table} d WHERE ${where.sql}`, + where.params + ) + const totalResults = Number(total.count) + if (page.limit === 0) return { results: [], totalResults } + const order = this.orderBy(sort) + const rows = await this.db.all( + `SELECT d."id", d."doc" FROM ${this.table} d + WHERE ${where.sql} ORDER BY ${order.sql} LIMIT ? OFFSET ?`, + [...where.params, ...order.params, page.limit, page.offset] + ) + return { results: rows.map(toDocument), totalResults } + } + + // the distinct values of a field (in lower case) in the documents that meet all + // conditions, with the number of documents for each, the most frequent first + async valueCounts( + paths: string[], + conditions: Condition[], + limit: number + ): Promise<{ value: string; count: number }[]> { + const where = this.where(conditions) + const rows = await this.db.all<{ value: string; count: number }>( + `SELECT f."str" AS "value", COUNT(DISTINCT f."id") AS "count" FROM ${this.fields} f + WHERE f."path" IN (${placeholders(paths.length)}) + AND f."id" IN (SELECT d."id" FROM ${this.table} d WHERE ${where.sql}) + GROUP BY f."str" ORDER BY "count" DESC, "value" ASC LIMIT ?`, + [...paths, ...where.params, limit] + ) + return rows.map((row) => ({ value: row.value, count: Number(row.count) })) + } + + // returns the number of deleted documents + deleteMatching(conditions: Condition[]): Promise { + const where = this.where(conditions) + const matching = `SELECT d."id" FROM ${this.table} d WHERE ${where.sql}` + return this.db.transaction(async (tx) => { + const ids = await tx.all<{ id: string }>(matching, where.params) + for (let start = 0; start < ids.length; start += CHUNK_SIZE) { + const chunk = ids.slice(start, start + CHUNK_SIZE).map((row) => row.id) + const list = placeholders(chunk.length) + await tx.run(`DELETE FROM ${this.table} WHERE "id" IN (${list})`, chunk) + await tx.run(`DELETE FROM ${this.fields} WHERE "id" IN (${list})`, chunk) + } + return ids.length + }) + } + + // ── one fixed SQL form for each kind of condition ───────────────────── + + private where(conditions: Condition[]): SqlPart { + return conditions.length > 0 + ? join( + conditions.map((condition) => this.condition(condition)), + ' AND ' + ) + : MATCH_ALL + } + + private condition(c: Condition): SqlPart { + if ('either' in c) { + if (c.either.length === 0) return MATCH_NONE + return wrap( + '(', + join( + c.either.map((part) => this.condition(part)), + ' OR ' + ), + ')' + ) + } + const inPaths = { + sql: `"path" IN (${placeholders(c.paths.length)})`, + params: c.paths + } + if ('anyOf' in c || 'noneOf' in c) { + const values = 'anyOf' in c ? c.anyOf : c.noneOf + if (values.length === 0) return 'anyOf' in c ? MATCH_NONE : MATCH_ALL + const texts = values.map(searchText) + // the key finds the rows in the index, the full text decides + const equal = this.hasField(inPaths, { + sql: `${KEY} IN (${placeholders(texts.length)}) AND "str" IN (${placeholders(texts.length)})`, + params: [...texts.map(keyOf), ...texts] + }) + return 'anyOf' in c ? equal : wrap('NOT (', equal, ')') + } + if ('range' in c) { + const bounds = Object.entries(c.range) + .filter(([, bound]) => bound !== undefined) + .map(([key, bound]) => + compare(RANGE_OPERATORS[key as keyof typeof RANGE_OPERATORS], bound) + ) + return this.hasField(inPaths, ...bounds) + } + if ('exists' in c) { + // the field itself, or a field below it + const below = c.paths.map(() => `"path" LIKE ? ESCAPE '\\'`).join(' OR ') + return this.hasField({ + sql: `(${inPaths.sql} OR ${below})`, + params: [...c.paths, ...c.paths.map((path) => `${escapeLike(path)}.%`)] + }) + } + // in brackets, so the words stay together inside an `either` + return wrap( + '(', + join( + c.words.map((word) => + this.hasField(inPaths, { + sql: `"str" LIKE ? ESCAPE '\\'`, + params: [`%${escapeLike(searchText(word))}%`] + }) + ), + ' AND ' + ), + ')' + ) + } + + // the documents that have a field row for which all parts hold + private hasField(...parts: SqlPart[]): SqlPart { + const row = join(parts, ' AND ') + return { + sql: `d."id" IN (SELECT "id" FROM ${this.fields} WHERE ${row.sql})`, + params: row.params + } + } + + // A document can have several values for a field: the lowest one counts for an + // ascending sort, the highest one for a descending sort. The number comes before the + // text, so a number field sorts as a number. A document without the field is last. + private orderBy(sort: SortSpec[]): SqlPart { + const parts = sort.map(({ paths, order }) => { + const [pick, direction] = order === 'desc' ? ['MAX', 'DESC'] : ['MIN', 'ASC'] + const value = (column: string) => + `(SELECT ${pick}("${column}") FROM ${this.fields} f + WHERE f."id" = d."id" AND f."path" IN (${placeholders(paths.length)})) + ${direction} NULLS LAST` + return { sql: `${value('num')}, ${value('str')}`, params: [...paths, ...paths] } + }) + return join([...parts, { sql: 'd."id" ASC', params: [] }], ', ') + } + + private async read(db: SqlClient, id: string, lock = ''): Promise { + const row = await db.get( + `SELECT "id", "doc" FROM ${this.table} WHERE "id" = ?${lock}`, + [id] + ) + return row && row.doc !== 'null' ? toDocument(row) : null + } + + private async write( + tx: SqlClient, + id: string, + doc: Document, + extraFields: Document + ): Promise { + await tx.run( + `INSERT INTO ${this.table} ("id", "doc") VALUES (?, ?) + ON CONFLICT ("id") DO UPDATE SET "doc" = excluded."doc"`, + [id, JSON.stringify(doc)] + ) + await tx.run(`DELETE FROM ${this.fields} WHERE "id" = ?`, [id]) + // the id is a field too, so a condition on `id` matches a document with no id field + const rows = flattenDocument({ id, ...doc, ...extraFields }) + for (let start = 0; start < rows.length; start += CHUNK_SIZE) { + await this.insertFields(tx, id, rows.slice(start, start + CHUNK_SIZE)) + } + } + + private async insertFields(tx: SqlClient, id: string, rows: FieldRow[]): Promise { + await tx.run( + `INSERT INTO ${this.fields} ("id", "path", "str", "num") + VALUES ${rows.map(() => '(?, ?, ?, ?)').join(', ')}`, + rows.flatMap((row) => [id, row.path, row.str, row.num]) + ) + } + + private async remove(tx: SqlClient, id: string): Promise { + await tx.run(`DELETE FROM ${this.fields} WHERE "id" = ?`, [id]) + const { changes } = await tx.run(`DELETE FROM ${this.table} WHERE "id" = ?`, [id]) + return changes === 1 + } +} + +// the document, with its id when the document has no id field +function toDocument(row: { id: string; doc: string }): Document { + const doc = JSON.parse(row.doc) + if (!doc.id) doc.id = row.id + return doc +} + +// A bound that is a number is compared as a number with the values that are numbers, and +// as text with the other values: an ISO date is compared as text. +function compare(operator: string, bound: Scalar): SqlPart { + const asText = { sql: `"str" ${operator} ?`, params: [searchText(bound)] } + const number = numberOf(bound) + if (number === null) return asText + return { + sql: `(("num" IS NOT NULL AND "num" ${operator} ?) OR ("num" IS NULL AND ${asText.sql}))`, + params: [number, ...asText.params] + } +} + +function escapeLike(value: string): string { + return value.replace(/[\\%_]/g, (character) => `\\${character}`) +} + +function join(parts: SqlPart[], separator: string): SqlPart { + return { + sql: parts.map((part) => part.sql).join(separator), + params: parts.flatMap((part) => part.params) + } +} + +function wrap(before: string, part: SqlPart, after: string): SqlPart { + return { sql: `${before}${part.sql}${after}`, params: part.params } +} diff --git a/src/components/database/sql/SqlEscrow.ts b/src/components/database/sql/SqlEscrow.ts new file mode 100644 index 000000000..44284f9d1 --- /dev/null +++ b/src/components/database/sql/SqlEscrow.ts @@ -0,0 +1,50 @@ +import type { EscrowEvent } from '../../../@types/Escrow.js' +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import { AbstractEscrowDatabase } from '../BaseDatabase.js' +import { escrowConditions, escrowPage } from '../escrowSearch.js' +import type { SqlClient } from './SqlClient.js' +import { SqlDocuments } from './SqlDocuments.js' +import { orNull } from './orNull.js' + +// the escrow events of the indexed chains: id = the event id +export class SqlEscrowDatabase extends AbstractEscrowDatabase { + private events: SqlDocuments + + constructor(config: OceanNodeDBConfig, db: SqlClient) { + super(config) + this.events = new SqlDocuments(db, 'escrow') + } + + init(): Promise { + return this.events.init() + } + + create(event: EscrowEvent) { + return orNull(`Error when creating escrow event ${event.id}`, async () => { + await this.events.put(event.id, event) + return event + }) + } + + retrieve(id: string) { + return orNull(`Error when retrieving escrow event ${id}`, () => this.events.get(id)) + } + + // a filter value is an equality value or a range on a number field + search(filters: Record, offset?: number, size?: number) { + const page = escrowPage(offset, size) + return orNull('Error when searching escrow events', async () => { + const { results } = await this.events.find(escrowConditions(filters), [], { + offset: page.from, + limit: page.limit + }) + return results + }) + } + + delete(id: string) { + return orNull(`Error when deleting escrow event ${id}`, async () => + (await this.events.delete(id)) ? { id } : null + ) + } +} diff --git a/src/components/database/sql/SqlIndexer.ts b/src/components/database/sql/SqlIndexer.ts new file mode 100644 index 000000000..ddebb3eb6 --- /dev/null +++ b/src/components/database/sql/SqlIndexer.ts @@ -0,0 +1,64 @@ +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import { AbstractIndexerDatabase } from '../BaseDatabase.js' +import type { SqlClient } from './SqlClient.js' +import { orNull } from './orNull.js' + +// the last indexed block of each chain: id = the chain id +export class SqlIndexerDatabase extends AbstractIndexerDatabase { + constructor( + config: OceanNodeDBConfig, + private db: SqlClient + ) { + super(config) + } + + init(): Promise { + return this.db.exec( + 'CREATE TABLE IF NOT EXISTS indexer ("id" TEXT PRIMARY KEY, "lastIndexedBlock" BIGINT)' + ) + } + + create(network: number, lastIndexedBlock: number) { + const id = network.toString() + return orNull( + `Error when writing the indexer entry of network ${network}`, + async () => { + await this.db.run( + `INSERT INTO indexer ("id", "lastIndexedBlock") VALUES (?, ?) + ON CONFLICT ("id") DO UPDATE SET "lastIndexedBlock" = excluded."lastIndexedBlock"`, + [id, lastIndexedBlock] + ) + return { id, lastIndexedBlock } + } + ) + } + + // null for a chain that is not indexed yet + retrieve(network: number) { + return orNull( + `Error when reading the indexer entry of network ${network}`, + async () => { + const row = await this.db.get<{ lastIndexedBlock: number }>( + 'SELECT "lastIndexedBlock" FROM indexer WHERE "id" = ?', + [network.toString()] + ) + return row ?? null + } + ) + } + + update(network: number, lastIndexedBlock: number) { + return this.create(network, lastIndexedBlock) + } + + delete(network: number) { + const id = network.toString() + return orNull( + `Error when deleting the indexer entry of network ${network}`, + async () => { + const { changes } = await this.db.run('DELETE FROM indexer WHERE "id" = ?', [id]) + return changes === 1 ? { id } : null + } + ) + } +} diff --git a/src/components/database/sql/SqlLog.ts b/src/components/database/sql/SqlLog.ts new file mode 100644 index 000000000..f48b43644 --- /dev/null +++ b/src/components/database/sql/SqlLog.ts @@ -0,0 +1,135 @@ +import { randomUUID } from 'node:crypto' +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import { DATABASE_LOGGER } from '../../../utils/logging/common.js' +import { LOG_LEVELS_STR } from '../../../utils/logging/Logger.js' +import { AbstractLogDatabase } from '../BaseDatabase.js' +import { Conditions, SqlClient, insertRow } from './SqlClient.js' + +const LOG_INSERT_FAILED_KEY = 'sql-log-insert-failed' +// the largest page of logs +const PAGE_SIZE = 250 + +type LogRow = { id: string; doc: string } + +const toLog = (row: LogRow): Record => ({ + ...JSON.parse(row.doc), + id: row.id +}) + +// The log lines of the node (LOG_DB). "timestamp" is an ISO date, which sorts as text. +// The module and the level have columns: the log API filters on them. +export class SqlLogDatabase extends AbstractLogDatabase { + constructor( + config: OceanNodeDBConfig, + private db: SqlClient + ) { + super(config) + } + + init(): Promise { + return this.db.exec(` + CREATE TABLE IF NOT EXISTS logs ( + "id" TEXT PRIMARY KEY, "timestamp" TEXT NOT NULL, "moduleName" TEXT, "level" TEXT, + "doc" TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS logs_timestamp ON logs ("timestamp");`) + } + + async insertLog(logEntry: Record) { + const id = randomUUID() + const timestamp = new Date().toISOString() + try { + await insertRow(this.db, 'logs', { + id, + timestamp, + moduleName: logEntry.moduleName, + level: logEntry.level, + doc: JSON.stringify({ ...logEntry, timestamp }) + }) + return { ...logEntry, id } + } catch (error) { + // one line for each failed write would double the output while the database is down + DATABASE_LOGGER.throttled( + LOG_INSERT_FAILED_KEY, + LOG_LEVELS_STR.LEVEL_ERROR, + `Error when inserting log entry: ${error.message}` + ) + return null + } + } + + async retrieveLog(id: string): Promise | null> { + try { + const row = await this.db.get( + 'SELECT "id", "doc" FROM logs WHERE "id" = ?', + [id] + ) + return row ? toLog(row) : null + } catch (error) { + DATABASE_LOGGER.error(`Error when retrieving log entry: ${error.message}`) + return null + } + } + + // Newest first. With `page` (starts at 0): one page of at most PAGE_SIZE logs. Without + // it: the first `maxLogs` logs. + async retrieveMultipleLogs( + startTime: Date, + endTime: Date, + maxLogs: number, + moduleName?: string, + level?: string, + page?: number + ): Promise[]> { + const where = new Conditions() + .add('"timestamp" >= ?', startTime.toISOString()) + .add('"timestamp" <= ?', endTime.toISOString()) + if (moduleName) where.add('LOWER("moduleName") = ?', moduleName.toLowerCase()) + if (level) where.add('LOWER("level") = ?', level.toLowerCase()) + const hasPage = page !== undefined && page !== null + const limit = hasPage ? Math.min(maxLogs, PAGE_SIZE) : maxLogs + try { + const rows = await this.db.all( + `SELECT "id", "doc" FROM logs${where} ORDER BY "timestamp" DESC LIMIT ? OFFSET ?`, + [...where.params, limit, hasPage ? page * limit : 0] + ) + return rows.map(toLog) + } catch (error) { + DATABASE_LOGGER.error( + `Error when retrieving multiple log entries: ${error.message}` + ) + return [] + } + } + + async delete(logId: string): Promise { + if (!logId) { + throw new Error('Log ID is required for deletion.') + } + await this.db.run('DELETE FROM logs WHERE "id" = ?', [logId]) + } + + // deletes the logs that are older than LOG_RETENTION_TIME (milliseconds) + async deleteOldLogs(): Promise { + const limit = new Date(Date.now() - this.retentionMs()).toISOString() + try { + const { changes } = await this.db.run('DELETE FROM logs WHERE "timestamp" <= ?', [ + limit + ]) + return changes + } catch (error) { + DATABASE_LOGGER.error(`Error when deleting old log entries: ${error.message}`) + return 0 + } + } + + async getLogsCount(): Promise { + try { + const row = await this.db.get('SELECT COUNT(*) AS "count" FROM logs') + return Number(row.count) + } catch (error) { + DATABASE_LOGGER.error(`Unable to retrieve logs count: ${error.message}`) + return 0 + } + } +} diff --git a/src/components/database/sql/SqlNodeMetrics.ts b/src/components/database/sql/SqlNodeMetrics.ts new file mode 100644 index 000000000..096226a94 --- /dev/null +++ b/src/components/database/sql/SqlNodeMetrics.ts @@ -0,0 +1,146 @@ +import type { + NodeMetricsHourly, + NodeMetricsSnapshot +} from '../../../@types/nodeMetrics.js' +import type { NodeMetricsProvider } from '../NodeMetricsDatabase.js' +import { + HOUR_MS, + HourlyRow, + SampleRow, + aggregateRows, + floorToHour, + hourlyRowToApi, + hourlyToRow, + snapshotToSampleRow +} from '../nodeMetricsAggregate.js' +import { SqlClient, insertRow } from './SqlClient.js' + +const SAMPLES = 'node_metrics_samples' +const HOURLY = 'node_metrics_hourly' +// The rows of an instance. A database from a release that had no instances has rows with +// no instance id: SQLite has one instance for each database, so they are its rows. +const OF_INSTANCE = '("instanceId" = ? OR "instanceId" IS NULL)' +// the samples of one hour of an instance: instanceId, hourStart, hourStart + 1 h +const IN_HOUR = `${OF_INSTANCE} AND "collectedAt" >= ? AND "collectedAt" < ?` + +// the number columns that the two tables share; "gpu" and "env" are JSON text +const METRIC_COLUMNS = [ + 'cpuUsagePercent', + 'coresAllocated', + 'hostCores', + 'throttledCount', + 'memUsedBytes', + 'memLimitBytes', + 'hostFreeBytes', + 'hostTotalBytes', + 'diskUsedBytes', + 'rxBytes', + 'txBytes', + 'jobsRunning', + 'jobsRunningFree', + 'jobsQueued', + 'jobsQueuedFree', + 'sampledContainers' +] +const METRIC_COLUMNS_SQL = [ + ...METRIC_COLUMNS.map((column) => `"${column}" DOUBLE PRECISION DEFAULT 0`), + `"gpu" TEXT DEFAULT '[]'`, + `"env" TEXT DEFAULT '[]'` +].join(', ') + +// Node metrics history: one sample per minute, averaged into one row per hour. +// Several instances can share the database, so each row has the id of its instance. +export class SqlNodeMetrics implements NodeMetricsProvider { + constructor(private db: SqlClient) {} + + init(): Promise { + return this.db.exec(` + CREATE TABLE IF NOT EXISTS ${SAMPLES} ( + "instanceId" TEXT, "collectedAt" BIGINT NOT NULL, ${METRIC_COLUMNS_SQL} + ); + CREATE INDEX IF NOT EXISTS idx_node_metrics_samples_instance + ON ${SAMPLES} ("instanceId", "collectedAt"); + CREATE TABLE IF NOT EXISTS ${HOURLY} ( + "instanceId" TEXT, "hourStart" BIGINT NOT NULL, "sampleCount" BIGINT DEFAULT 0, + ${METRIC_COLUMNS_SQL}, + PRIMARY KEY ("instanceId", "hourStart") + );`) + } + + async insertSample(instanceId: string, snapshot: NodeMetricsSnapshot): Promise { + await insertRow(this.db, SAMPLES, { instanceId, ...snapshotToSampleRow(snapshot) }) + } + + async getPendingSampleHours(instanceId: string, beforeHour: number): Promise { + const rows = await this.db.all<{ collectedAt: number }>( + `SELECT "collectedAt" FROM ${SAMPLES} WHERE ${OF_INSTANCE} AND "collectedAt" < ?`, + [instanceId, beforeHour] + ) + const hours = new Set(rows.map((row) => floorToHour(row.collectedAt))) + return [...hours].sort((a, b) => a - b) + } + + // Averages the samples of the hour into one hourly row, then deletes them. One + // transaction: a failure must not lose the samples or count them twice. + rollupHour(instanceId: string, hourStart: number): Promise { + const hour = [instanceId, hourStart] + return this.db.transaction(async (tx) => { + // the samples that are deleted are the samples that are counted: a sample that + // comes during the roll-up stays for the next one + const samples = await tx.all( + `DELETE FROM ${SAMPLES} WHERE ${IN_HOUR} RETURNING *`, + [...hour, hourStart + HOUR_MS] + ) + if (samples.length === 0) return false + await tx.run(`DELETE FROM ${HOURLY} WHERE ${OF_INSTANCE} AND "hourStart" = ?`, hour) + await insertRow(tx, HOURLY, { + instanceId, + ...hourlyToRow(aggregateRows(samples, hourStart)) + }) + return true + }) + } + + async getPartialHour( + instanceId: string, + hourStart: number + ): Promise { + const samples = await this.db.all( + `SELECT * FROM ${SAMPLES} WHERE ${IN_HOUR}`, + [instanceId, hourStart, hourStart + HOUR_MS] + ) + if (samples.length === 0) return null + return { ...aggregateRows(samples, hourStart), partial: true } + } + + async purgeOldSamples(instanceId: string, olderThan: number): Promise { + const { changes } = await this.db.run( + `DELETE FROM ${SAMPLES} WHERE ${OF_INSTANCE} AND "collectedAt" < ?`, + [instanceId, olderThan] + ) + return changes + } + + async deleteHourlyOlderThan(instanceId: string, cutoff: number): Promise { + const { changes } = await this.db.run( + `DELETE FROM ${HOURLY} WHERE ${OF_INSTANCE} AND "hourStart" < ?`, + [instanceId, cutoff] + ) + return changes + } + + async getHourly( + instanceId: string, + from: number, + to: number, + limit: number + ): Promise { + const rows = await this.db.all( + `SELECT * FROM ${HOURLY} + WHERE ${OF_INSTANCE} AND "hourStart" >= ? AND "hourStart" <= ? + ORDER BY "hourStart" ASC LIMIT ?`, + [instanceId, from, to, limit] + ) + return rows.map(hourlyRowToApi) + } +} diff --git a/src/components/database/sql/SqlNonce.ts b/src/components/database/sql/SqlNonce.ts new file mode 100644 index 000000000..7ee1fc7b2 --- /dev/null +++ b/src/components/database/sql/SqlNonce.ts @@ -0,0 +1,78 @@ +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import { addressCasingVariants, normalizeAddress } from '../../../utils/evmAddress.js' +import { AbstractNonceDatabase } from '../BaseDatabase.js' +import { orNull } from './orNull.js' +import { SqlClient, placeholders } from './SqlClient.js' + +// one row per address: id = the checksummed address +export class SqlNonceDatabase extends AbstractNonceDatabase { + constructor( + config: OceanNodeDBConfig, + private db: SqlClient + ) { + super(config) + } + + init(): Promise { + return this.db.exec( + 'CREATE TABLE IF NOT EXISTS nonce ("id" TEXT PRIMARY KEY, "nonce" BIGINT)' + ) + } + + create(address: string, nonce: number) { + return orNull(`Error when writing nonce ${nonce} for address ${address}`, () => + this.write(address, nonce) + ) + } + + // Reads every casing of the address and keeps the HIGHEST nonce: a row from before the + // addresses were normalized must not make the nonce go back, which would open the replay + // window again. A failed read throws: the callers must not take it for "no nonce yet". + async retrieve(address: string) { + try { + const variants = addressCasingVariants(address) + const rows = await this.db.all<{ nonce: number }>( + `SELECT "nonce" FROM nonce WHERE "id" IN (${placeholders(variants.length)})`, + variants + ) + const nonces = rows.map((row) => row.nonce).filter((n) => typeof n === 'number') + return { + id: normalizeAddress(address), + nonce: nonces.length > 0 ? Math.max(...nonces) : null + } + } catch (error) { + this.logError(`Error when retrieving the nonce of address ${address}`, error) + throw error + } + } + + update(address: string, nonce: number) { + return orNull(`Error when updating nonce ${nonce} for address ${address}`, () => + this.write(address, nonce, 'WHERE nonce."nonce" < excluded."nonce"') + ) + } + + delete(address: string) { + const id = normalizeAddress(address) + return orNull(`Error when deleting the nonce of address ${address}`, async () => { + const row = await this.db.get<{ nonce: number }>( + 'SELECT "nonce" FROM nonce WHERE "id" = ?', + [id] + ) + if (!row) return { id, nonce: null } + await this.db.run('DELETE FROM nonce WHERE "id" = ?', [id]) + return { id, nonce: row.nonce } + }) + } + + // null when the condition kept the stored row + private async write(address: string, nonce: number, condition = '') { + const id = normalizeAddress(address) + const { changes } = await this.db.run( + `INSERT INTO nonce ("id", "nonce") VALUES (?, ?) + ON CONFLICT ("id") DO UPDATE SET "nonce" = excluded."nonce" ${condition}`, + [id, nonce] + ) + return changes > 0 ? { id, nonce } : null + } +} diff --git a/src/components/database/sql/SqlOrder.ts b/src/components/database/sql/SqlOrder.ts new file mode 100644 index 000000000..a747e7417 --- /dev/null +++ b/src/components/database/sql/SqlOrder.ts @@ -0,0 +1,90 @@ +import lodash from 'lodash' +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import { AbstractOrderDatabase, Found, OrderOwner, Page } from '../BaseDatabase.js' +import type { SqlClient } from './SqlClient.js' +import { SqlDocuments } from './SqlDocuments.js' +import { orNull } from './orNull.js' + +// the orders of the indexed chains: id = the order transaction +export class SqlOrderDatabase extends AbstractOrderDatabase { + private orders: SqlDocuments + + constructor(config: OceanNodeDBConfig, db: SqlClient) { + super(config) + // "order" is a reserved word in SQL + this.orders = new SqlDocuments(db, 'orders') + } + + init(): Promise { + return this.orders.init() + } + + find(owner: OrderOwner, page: Page): Promise { + const [field, value] = Object.entries(owner)[0] + return this.orders.find( + [{ paths: [field], anyOf: [value] }], + [{ paths: ['timestamp'], order: 'desc' }], + page + ) + } + + create( + orderId: string, + type: string, + timestamp: number, + consumer: string, + payer: string, + datatokenAddress: string, + nftAddress: string, + did: string, + startOrderId?: string + ) { + const order = { + orderId, + type, + timestamp, + consumer, + payer, + datatokenAddress, + nftAddress, + did, + startOrderId + } + return orNull(`Error when creating order ${orderId}`, async () => { + await this.orders.put(orderId, order) + return order + }) + } + + retrieve(orderId: string) { + return orNull(`Error when retrieving order ${orderId}`, () => + this.orders.get(orderId) + ) + } + + // changes the given fields only; creates the order when it does not exist + update( + orderId: string, + type: string, + timestamp: number, + consumer: string, + payer: string, + startOrderId?: string, + datatokenAddress?: string + ) { + const changes = { type, timestamp, consumer, payer, datatokenAddress, startOrderId } + return orNull(`Error when updating order ${orderId}`, async () => { + await this.orders.modify(orderId, (order) => ({ + ...order, + ...lodash.omitBy(changes, lodash.isUndefined) + })) + return changes + }) + } + + delete(orderId: string) { + return orNull(`Error when deleting order ${orderId}`, async () => + (await this.orders.delete(orderId)) ? { id: orderId } : null + ) + } +} diff --git a/src/components/database/sql/SqlStores.ts b/src/components/database/sql/SqlStores.ts new file mode 100644 index 000000000..d937a2732 --- /dev/null +++ b/src/components/database/sql/SqlStores.ts @@ -0,0 +1,79 @@ +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import { AuthTokenDatabase } from '../AuthTokenDatabase.js' +import { C2DDatabase } from '../C2DDatabase.js' +import type { DatabaseStores } from '../DatabaseFactory.js' +import type { SqlClient } from './SqlClient.js' +import { SqlAuthToken } from './SqlAuthToken.js' +import { SqlBucketRegistry } from './SqlBucketRegistry.js' +import { SqlCompute } from './SqlCompute.js' +import { SqlConfigDatabase } from './SqlConfig.js' +import { SqlNodeMetrics } from './SqlNodeMetrics.js' +import { SqlNonceDatabase } from './SqlNonce.js' +import { SqlAccessListDatabase } from './SqlAccessList.js' +import { SqlDdoDatabase } from './SqlDdo.js' +import { SqlDdoStateDatabase } from './SqlDdoState.js' +import { SqlEscrowDatabase } from './SqlEscrow.js' +import { SqlIndexerDatabase } from './SqlIndexer.js' +import { SqlLogDatabase } from './SqlLog.js' +import { SqlOrderDatabase } from './SqlOrder.js' + +// The stores are in groups. PostgreSQL has one database for all groups; SQLite has one +// file for each group. +export type SqlStateStores = Pick< + DatabaseStores, + 'nonce' | 'config' | 'c2d' | 'authToken' | 'buckets' | 'nodeMetrics' +> +export type SqlIndexStores = Omit + +export type SqlStoreGroup = keyof SqlStateStores | 'index' +export type ClientFor = (group: SqlStoreGroup) => SqlClient + +// creates the tables of a store and gives the store +async function ready }>(store: T): Promise { + await store.init() + return store +} + +// the stores of the node state: jobs, services, nonces, tokens, settings, buckets, metrics +export async function buildSqlStateStores( + config: OceanNodeDBConfig, + clientFor: ClientFor +): Promise { + return { + nonce: await ready(new SqlNonceDatabase(config, clientFor('nonce'))), + config: await ready(new SqlConfigDatabase(clientFor('config'))), + c2d: new C2DDatabase(config, await ready(new SqlCompute(clientFor('c2d')))), + authToken: new AuthTokenDatabase( + config, + await ready(new SqlAuthToken(clientFor('authToken'))) + ), + buckets: await ready(new SqlBucketRegistry(clientFor('buckets'))), + nodeMetrics: await ready(new SqlNodeMetrics(clientFor('nodeMetrics'))) + } +} + +// the stores that the indexer fills from the chains, and the logs +export async function buildSqlIndexStores( + config: OceanNodeDBConfig, + db: SqlClient +): Promise { + return { + ddo: await ready(new SqlDdoDatabase(config, db)), + indexer: await ready(new SqlIndexerDatabase(config, db)), + logs: await ready(new SqlLogDatabase(config, db)), + order: await ready(new SqlOrderDatabase(config, db)), + ddoState: await ready(new SqlDdoStateDatabase(config, db)), + accessList: await ready(new SqlAccessListDatabase(config, db)), + escrow: await ready(new SqlEscrowDatabase(config, db)) + } +} + +export async function buildSqlStores( + config: OceanNodeDBConfig, + clientFor: ClientFor +): Promise { + return { + ...(await buildSqlStateStores(config, clientFor)), + ...(await buildSqlIndexStores(config, clientFor('index'))) + } +} diff --git a/src/components/database/sql/SqliteClient.ts b/src/components/database/sql/SqliteClient.ts new file mode 100644 index 000000000..b1e262bac --- /dev/null +++ b/src/components/database/sql/SqliteClient.ts @@ -0,0 +1,105 @@ +import { DatabaseSync } from 'node:sqlite' +import fs from 'fs' +import path from 'path' +import { SqlClient, toSqlValue } from './SqlClient.js' + +const IN_MEMORY = ':memory:' + +type Bindable = string | number | bigint | Uint8Array | null + +const bind = (params: unknown[]) => params.map(toSqlValue) as Bindable[] + +// runs a synchronous task and gives its result, or its throw, as a promise +function settle(task: () => T): Promise { + return new Promise((resolve) => resolve(task())) +} + +/** + * SQLite through the built-in `node:sqlite`, one handle per file. + * + * `node:sqlite` is synchronous: a statement runs on the main thread. The calls go through a + * queue, so the statements of a transaction are never mixed with the statements of another + * caller that runs between two `await`s. + */ +export class SqliteClient implements SqlClient { + // a transaction holds the write lock of the database from its start + readonly rowLock = '' + // the default of SQLite + readonly byteOrderText = 'TEXT' + private db: DatabaseSync + private queue: Promise = Promise.resolve() + // the client that a transaction gives to its work: it runs at once, inside the queue slot + // that the transaction holds + private readonly direct: SqlClient = { + rowLock: this.rowLock, + byteOrderText: this.byteOrderText, + exec: (sql) => settle(() => this.db.exec(sql)), + run: (sql, params = []) => settle(() => this.runNow(sql, params)), + get: (sql, params = []) => + settle(() => this.db.prepare(sql).get(...bind(params)) as any), + all: (sql, params = []) => + settle(() => this.db.prepare(sql).all(...bind(params)) as any[]), + transaction: (work) => work(this.direct), + close: () => settle(() => this.db.close()) + } + + constructor(dbFilePath: string) { + if (dbFilePath !== IN_MEMORY) { + // DatabaseSync throws when the folder of the file does not exist + fs.mkdirSync(path.dirname(dbFilePath), { recursive: true }) + } + this.db = new DatabaseSync(dbFilePath) + // WAL: a reader does not block the writer. busy_timeout: wait for a lock of another + // process (for example the sqlite3 CLI on a live file) before the statement fails. + this.db.exec('PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;') + } + + exec(sql: string): Promise { + return this.enqueue(() => this.direct.exec(sql)) + } + + run(sql: string, params?: unknown[]): Promise<{ changes: number }> { + return this.enqueue(() => this.direct.run(sql, params)) + } + + get(sql: string, params?: unknown[]): Promise { + return this.enqueue(() => this.direct.get(sql, params)) + } + + all(sql: string, params?: unknown[]): Promise { + return this.enqueue(() => this.direct.all(sql, params)) + } + + // IMMEDIATE takes the write lock at the start, so a second process that shares the file + // cannot commit between a read and a write of this transaction + transaction(work: (tx: SqlClient) => Promise): Promise { + return this.enqueue(async () => { + this.db.exec('BEGIN IMMEDIATE') + try { + const result = await work(this.direct) + this.db.exec('COMMIT') + return result + } catch (error) { + // SQLite ends the transaction itself on some errors (a full disk, for example) + if (this.db.isTransaction) this.db.exec('ROLLBACK') + throw error + } + }) + } + + close(): Promise { + return this.enqueue(() => this.direct.close()) + } + + private runNow(sql: string, params: unknown[]): { changes: number } { + const result = this.db.prepare(sql).run(...bind(params)) + return { changes: Number(result.changes) } + } + + // the task starts when all the tasks before it are done, whatever their result + private enqueue(task: () => Promise): Promise { + const result = this.queue.then(task) + this.queue = result.catch(() => {}) + return result + } +} diff --git a/src/components/database/sql/SqliteStores.ts b/src/components/database/sql/SqliteStores.ts new file mode 100644 index 000000000..cb71c2f3f --- /dev/null +++ b/src/components/database/sql/SqliteStores.ts @@ -0,0 +1,129 @@ +import path from 'path' +import type { OceanNodeDBConfig } from '../../../@types/OceanNode.js' +import type { DatabaseStores } from '../DatabaseFactory.js' +import { create256Hash } from '../../../utils/crypt.js' +import { DATABASE_LOGGER } from '../../../utils/logging/common.js' +import type { SqlClient } from './SqlClient.js' +import { parseBody } from './SqlCompute.js' +import { SqlStoreGroup, buildSqlStores } from './SqlStores.js' +import { SqliteClient } from './SqliteClient.js' + +const SQLITE_DIR = 'databases' + +// The file of each store group. They are the files of the releases before the pluggable +// database, so a node that updates keeps its state. +const FILES: Record = { + nonce: 'nonceDatabase.sqlite', + config: 'config.sqlite', + c2d: 'c2dDatabase.sqlite', + authToken: 'authTokenDatabase.sqlite', + buckets: 'persistentStorage.sqlite', + nodeMetrics: 'nodeMetrics.sqlite', + // the index data and the logs + index: 'index.sqlite' +} + +export type SqliteClients = Record + +export function openSqliteClients(dir: string): SqliteClients { + const clients = {} as SqliteClients + for (const [group, file] of Object.entries(FILES)) { + clients[group as SqlStoreGroup] = new SqliteClient(path.join(dir, file)) + } + return clients +} + +export async function createSqliteStores( + config: OceanNodeDBConfig, + dir: string = SQLITE_DIR +): Promise { + const clients = openSqliteClients(dir) + try { + await upgradeLegacyTables(clients) + return await buildSqlStores(config, (group) => clients[group]) + } catch (error) { + // the node tries again with new handles + await Promise.all(Object.values(clients).map((client) => client.close())) + throw error + } +} + +// a stored token id is the sha256 of the token +const HASHED_TOKEN = /^0x[0-9a-f]{64}$/ + +// Brings the tables of an older release to the form that the SQL stores use. Each step +// does nothing on a new database, and nothing when it runs again. +export async function upgradeLegacyTables(clients: SqliteClients): Promise { + const metrics = clients.nodeMetrics + await addColumnIfMissing(metrics, 'node_metrics_samples', 'instanceId', 'TEXT') + await addColumnIfMissing(metrics, 'node_metrics_hourly', 'instanceId', 'TEXT') + await addColumnIfMissing(clients.buckets, 'persistent_storage_buckets', 'label', 'TEXT') + await addColumnIfMissing(clients.authToken, 'authTokens', 'chainId', 'TEXT') + await hashPlainTokens(clients.authToken) + await addColumnIfMissing(clients.c2d, 'service_jobs', 'outputArchiveCount', 'BIGINT') + await countOutputArchives(clients.c2d) +} + +// the columns of a table; none when the table does not exist yet +async function columnsOf(db: SqlClient, table: string): Promise { + const columns = await db.all<{ name: string }>(`PRAGMA table_info("${table}")`) + return columns.map((column) => column.name) +} + +async function addColumnIfMissing( + db: SqlClient, + table: string, + column: string, + type: string +): Promise { + const columns = await columnsOf(db, table) + if (columns.length > 0 && !columns.includes(column)) { + await db.exec(`ALTER TABLE "${table}" ADD COLUMN "${column}" ${type}`) + } +} + +// the older releases kept the token itself as the key +function hashPlainTokens(db: SqlClient): Promise { + return db.transaction(async (tx) => { + const tableExists = (await columnsOf(tx, 'authTokens')).length > 0 + if (!tableExists) return + const rows = await tx.all<{ token: string }>('SELECT "token" FROM "authTokens"') + for (const { token } of rows.filter((row) => !HASHED_TOKEN.test(row.token))) { + await tx.run('UPDATE "authTokens" SET "token" = ? WHERE "token" = ?', [ + create256Hash(token), + token + ]) + } + }) +} + +// The number of output archives was only in the body. Every row that has no count gets +// one, so a start that follows an upgrade that did not end does the rest. +function countOutputArchives(db: SqlClient): Promise { + return db.transaction(async (tx) => { + if (!(await columnsOf(tx, 'service_jobs')).includes('outputArchiveCount')) return + const rows = await tx.all( + 'SELECT "serviceId", "body" FROM service_jobs WHERE "outputArchiveCount" IS NULL' + ) + for (const row of rows) { + await tx.run( + 'UPDATE service_jobs SET "outputArchiveCount" = ? WHERE "serviceId" = ?', + [outputArchiveCount(row), row.serviceId] + ) + } + }) +} + +function outputArchiveCount(row: { + serviceId: string + body: string | Uint8Array +}): number { + try { + return (parseBody(row.body).outputArchives ?? []).length + } catch (error) { + DATABASE_LOGGER.warn( + `Service job ${row.serviceId} has a body that is not JSON: ${error.message}` + ) + return 0 + } +} diff --git a/src/components/database/sql/orNull.ts b/src/components/database/sql/orNull.ts new file mode 100644 index 000000000..a1d7be47a --- /dev/null +++ b/src/components/database/sql/orNull.ts @@ -0,0 +1,15 @@ +import { DATABASE_LOGGER } from '../../../utils/logging/common.js' +import { QueryError } from '../BaseDatabase.js' + +// The callers of these stores expect null, not a throw, when the database fails. The +// failure is logged here. A query that the node cannot run is an error of the caller, +// not of the database: it is thrown. +export async function orNull(what: string, work: () => Promise): Promise { + try { + return await work() + } catch (error) { + if (error instanceof QueryError) throw error + DATABASE_LOGGER.error(`${what}: ${error.message}`) + return null + } +} diff --git a/src/components/database/sqlite.ts b/src/components/database/sqlite.ts deleted file mode 100644 index 4a081e55d..000000000 --- a/src/components/database/sqlite.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { TypesenseSchema, typesenseSchemas } from './TypesenseSchemas.js' -import { SqliteClient } from './sqliteClient.js' - -interface DatabaseProvider { - createNonce(address: string, nonce: number): Promise<{ id: string; nonce: number }> - retrieveNonce(address: string): Promise<{ id: string; nonce: number | null }> - updateNonce(address: string, nonce: number): Promise<{ id: string; nonce: number }> - deleteNonceEntry(address: string): Promise<{ id: string; nonce: number | null }> -} - -export class SQLiteProvider implements DatabaseProvider { - private db: SqliteClient - private schemaNonce: TypesenseSchema - private configSchema: string - - constructor(dbFilePath: string) { - this.db = new SqliteClient(dbFilePath) - this.schemaNonce = typesenseSchemas.nonceSchemas - this.configSchema = 'config' - } - - // eslint-disable-next-line require-await - async createTableForNonce() { - this.db.exec(` - CREATE TABLE IF NOT EXISTS ${this.schemaNonce.name} ( - id TEXT PRIMARY KEY, - nonce INTEGER - ); - `) - } - - // eslint-disable-next-line require-await - async createTableForConfig() { - this.db.exec(` - CREATE TABLE IF NOT EXISTS ${this.configSchema} ( - key TEXT NOT NULL PRIMARY KEY, - value TEXT - ); - `) - } - - // eslint-disable-next-line require-await - async createNonce(address: string, nonce: number) { - const insertSQL = ` - INSERT INTO ${this.schemaNonce.name} (id, nonce) - VALUES (?, ?) - ON CONFLICT(id) DO UPDATE SET nonce=excluded.nonce; - ` - this.db.run(insertSQL, [address, nonce]) - return { id: address, nonce } - } - - // eslint-disable-next-line require-await - async createOrUpdateConfig(key: string, value: string) { - const insertSQL = ` - INSERT INTO ${this.configSchema} (key, value) - VALUES (?, ?) - ON CONFLICT(key) DO UPDATE SET value = excluded.value; - ` - this.db.run(insertSQL, [key, value]) - return { key, value } - } - - // eslint-disable-next-line require-await - async retrieveNonce(address: string) { - const selectSQL = ` - SELECT * FROM ${this.schemaNonce.name} WHERE id = ? - ` - const row = this.db.get<{ nonce: number }>(selectSQL, [address]) - return row ? { id: address, nonce: row.nonce } : { id: address, nonce: null } - } - - // eslint-disable-next-line require-await - async retrieveValue(key: string) { - const selectSQL = ` - SELECT value FROM ${this.configSchema} WHERE key = ?; - ` - const row = this.db.get<{ value: string }>(selectSQL, [key]) - // Returns null if no version exists - return row ? { value: row.value } : { value: null } - } - - // eslint-disable-next-line require-await - async updateNonce(address: string, nonce: number) { - return this.createNonce(address, nonce) - } - - // eslint-disable-next-line require-await - async deleteNonceEntry(address: string) { - const selectSQL = ` - SELECT nonce FROM ${this.schemaNonce.name} WHERE id = ? - ` - - const deleteSQL = ` - DELETE FROM ${this.schemaNonce.name} WHERE id = ? - ` - - const row = this.db.get<{ nonce: number }>(selectSQL, [address]) - if (!row) return { id: address, nonce: null } - - this.db.run(deleteSQL, [address]) - return { id: address, nonce: row.nonce } - } -} diff --git a/src/components/database/sqliteAuthToken.ts b/src/components/database/sqliteAuthToken.ts deleted file mode 100644 index e50979c09..000000000 --- a/src/components/database/sqliteAuthToken.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { AuthToken } from './AuthTokenDatabase.js' -import { SqliteClient } from './sqliteClient.js' -import { DATABASE_LOGGER } from '../../utils/logging/common.js' - -interface AuthTokenDatabaseProvider { - createToken( - token: string, - address: string, - createdAt: number, - validUntil: number | null, - chainId?: string | null - ): Promise - validateTokenEntry(token: string): Promise - invalidateTokenEntry(token: string): Promise -} - -export class SQLiteAuthToken implements AuthTokenDatabaseProvider { - private db: SqliteClient - - constructor(dbFilePath: string) { - this.db = new SqliteClient(dbFilePath) - } - - // eslint-disable-next-line require-await - async createTable(): Promise { - this.db.exec(` - CREATE TABLE IF NOT EXISTS authTokens ( - token TEXT PRIMARY KEY, - address TEXT NOT NULL, - createdAt DATETIME DEFAULT CURRENT_TIMESTAMP, - validUntil DATETIME, - isValid BOOLEAN DEFAULT TRUE, - chainId TEXT - ) - `) - - // Migration for DBs created before the chainId column existed: add it if missing. - // A fresh table already has the column, so ALTER throws "duplicate column name" — - // that's the only expected failure, so swallow it. With the synchronous exec above this - // now runs strictly after the CREATE TABLE, unlike the old callback-based `await exec` - // which never actually waited and let the CREATE race the ALTER. - try { - this.db.exec(`ALTER TABLE authTokens ADD COLUMN chainId TEXT`) - } catch { - // column already exists - } - } - - // eslint-disable-next-line require-await - async createToken( - token: string, - address: string, - createdAt: number, - validUntil: number | null = null, - chainId?: string | null - ): Promise { - const insertSQL = ` - INSERT INTO authTokens (token, address, createdAt, validUntil, chainId) VALUES (?, ?, ?, ?, ?) - ` - try { - this.db.run(insertSQL, [token, address, createdAt, validUntil, chainId]) - } catch (err) { - DATABASE_LOGGER.error(`Error creating auth token: ${err}`) - throw err - } - } - - async validateTokenEntry(token: string): Promise { - const selectSQL = ` - SELECT * FROM authTokens WHERE token = ? - ` - let row: AuthToken | undefined - try { - row = this.db.get(selectSQL, [token]) - } catch (err) { - DATABASE_LOGGER.error(`Error validating auth token: ${err}`) - throw err - } - - if (!row) { - return null - } - - if (!row.isValid) { - return null - } - - if (row.validUntil === null) { - return row - } - - const validUntilDate = new Date(row.validUntil).getTime() - const now = Date.now() - - if (validUntilDate < now) { - DATABASE_LOGGER.debug('Expired auth token invalidated') - await this.invalidateTokenEntry(token) - return null - } - - return row - } - - // eslint-disable-next-line require-await - async invalidateTokenEntry(token: string): Promise { - const deleteSQL = ` - UPDATE authTokens SET isValid = FALSE WHERE token = ? - ` - try { - this.db.run(deleteSQL, [token]) - } catch (err) { - DATABASE_LOGGER.error(`Error invalidating auth token: ${err}`) - throw err - } - } -} diff --git a/src/components/database/sqliteClient.ts b/src/components/database/sqliteClient.ts deleted file mode 100644 index bd9a31f84..000000000 --- a/src/components/database/sqliteClient.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { DatabaseSync } from 'node:sqlite' -import fs from 'fs' -import path from 'path' - -type Bindable = string | number | bigint | Uint8Array | null - -// node:sqlite refuses `undefined` and boolean bindings (throws ERR_INVALID_ARG_TYPE); -// the old `sqlite3` package silently coerced them. Keep that lenient behaviour centrally -// so every call site binds the same way it did before the engine swap. -function sanitize(v: unknown): Bindable { - if (v === undefined) return null - if (typeof v === 'boolean') return v ? 1 : 0 - return v as Bindable -} - -/** - * Thin synchronous wrapper around Node's built-in `node:sqlite` (`DatabaseSync`). - * Owns a single database handle per file and the sqlite3-compatibility concerns - * (bind sanitization, eager parent-directory creation). This replaces the former - * `sqlite3` native addon; all embedded DBs (nonce, config, C2D jobs, auth tokens, - * persistent-storage registry) go through this client. - * - * The queries in this project are all single-row or small local-table operations, so - * running them synchronously on the main thread is acceptable. Do not point this client - * at large or unbounded datasets without reconsidering. - */ -export class SqliteClient { - private db: DatabaseSync - - constructor(dbFilePath: string) { - // DatabaseSync opens eagerly and throws synchronously if the parent directory is - // missing (unlike the old sqlite3.Database, which deferred opening). mkdir here so - // every DB path is covered uniformly and callers no longer need to mkdir themselves. - fs.mkdirSync(path.dirname(dbFilePath), { recursive: true }) - this.db = new DatabaseSync(dbFilePath) - // Long-running server process: wait briefly rather than fail if the DB is momentarily - // locked (e.g. an external `sqlite3` CLI reading a live file). - this.db.exec('PRAGMA busy_timeout = 5000;') - } - - exec(sql: string): void { - this.db.exec(sql) - } - - run(sql: string, params: unknown[] = []): { changes: number } { - const result = this.db.prepare(sql).run(...params.map(sanitize)) - // readBigInts is not enabled, so `changes` is a JS number already; coerce to keep the - // public type a plain number (all counts here are tiny, well below Number.MAX_SAFE_INTEGER). - return { changes: Number(result.changes) } - } - - get(sql: string, params: unknown[] = []): T | undefined { - return this.db.prepare(sql).get(...params.map(sanitize)) as T | undefined - } - - all(sql: string, params: unknown[] = []): T[] { - return this.db.prepare(sql).all(...params.map(sanitize)) as T[] - } -} diff --git a/src/components/database/sqliteCompute.ts b/src/components/database/sqliteCompute.ts deleted file mode 100644 index 900cde15b..000000000 --- a/src/components/database/sqliteCompute.ts +++ /dev/null @@ -1,1012 +0,0 @@ -import { typesenseSchemas, TypesenseSchema } from './TypesenseSchemas.js' -import { - C2DStatusNumber, - C2DStatusText, - type DBComputeJob, - type ContainerMetricsSnapshot -} from '../../@types/C2D/C2D.js' -import { - ServiceStatusNumber, - SERVICE_START_PENDING_STATUSES, - SERVICE_STOPPED_STATUSES, - isStoppedLike, - type ServiceJob, - type ServiceModelDownload, - type ServiceReadiness -} from '../../@types/C2D/ServiceOnDemand.js' -import { SqliteClient } from './sqliteClient.js' -import { DATABASE_LOGGER } from '../../utils/logging/common.js' -import { create256Hash } from '../../utils/crypt.js' - -interface ComputeDatabaseProvider { - newJob(job: DBComputeJob): Promise - getJob(jobId?: string, agreementId?: string, owner?: string): Promise - updateJob(job: DBComputeJob): Promise - getRunningJobs(engine?: string, environment?: string): Promise - deleteJob(jobId: string): Promise - getFinishedJobs(environments?: string[]): Promise - getJobs( - environments?: string[], - fromTimestamp?: number, - consumerAddrs?: string[], - status?: C2DStatusNumber, - runningJobs?: boolean - ): Promise - updateImage(image: string): Promise - getOldImages(retentionDays: number): Promise -} - -function getInternalStructure(job: DBComputeJob): any { - const internalBlob = { - clusterHash: job.clusterHash, - configlogURL: job.configlogURL, - publishlogURL: job.publishlogURL, - algologURL: job.algologURL, - outputsURL: job.outputsURL, - stopRequested: job.stopRequested, - algorithm: job.algorithm, - assets: job.assets, - isRunning: job.isRunning, - isStarted: job.isStarted, - containerImage: job.containerImage, - resources: job.resources, - isFree: job.isFree, - algoStartTimestamp: job.algoStartTimestamp, - algoStopTimestamp: job.algoStopTimestamp, - metadata: job.metadata, - additionalViewers: job.additionalViewers, - terminationDetails: job.terminationDetails, - payment: job.payment, - algoDuration: job.algoDuration, - queueMaxWaitTime: job.queueMaxWaitTime, - output: job.output, - outputBucketId: job.outputBucketId, - jobIdHash: job.jobIdHash, - buildStartTimestamp: job.buildStartTimestamp, - buildStopTimestamp: job.buildStopTimestamp, - // Runtime metrics snapshot. It MUST round-trip through the body blob: it is what the owner - // reads back on COMPUTE_GET_STATUS, and it is also the previous-sample accumulator every - // next sample needs — without it persisted, every sample is a "first sample", so CPU % can - // never be computed (no delta) and the memory peak never accumulates. Still node-internal - // on the way out: omitDBComputeFieldsFromComputeJob strips it unless the verified owner - // asked for it, so it stays out of the escrow claim proof. - runtimeMetrics: job.runtimeMetrics - } - return internalBlob -} -export function generateBlobFromJSON(job: DBComputeJob): Buffer { - return Buffer.from(JSON.stringify(getInternalStructure(job))) -} - -export function generateJSONFromBlob(blob: any): Promise { - // node:sqlite returns BLOB columns as Uint8Array (the old sqlite3 addon returned Buffer). - // Buffer.from() handles both, so decode through it before parsing. - return JSON.parse(Buffer.from(blob).toString()) -} - -// we cannot store array of strings, so we use string separators instead -export const STRING_SEPARATOR = '__,__' - -export function convertArrayToString(array: string[]) { - let str: string = '' - for (let i = 0; i < array.length; i++) { - str = str + array[i] - // Do not append comma at the end of last element - if (i < array.length - 1) { - str = str + STRING_SEPARATOR - } - } - return str -} -export function convertStringToArray(str: string) { - const arr: string[] = str.split(STRING_SEPARATOR) - return arr -} - -export class SQLiteCompute implements ComputeDatabaseProvider { - private db: SqliteClient - private schema: TypesenseSchema - - constructor(dbFilePath: string) { - this.db = new SqliteClient(dbFilePath) - this.schema = typesenseSchemas.c2dSchemas - } - - // eslint-disable-next-line require-await - async deleteJob(jobId: string): Promise { - const deleteSQL = ` - DELETE FROM ${this.schema.name} WHERE jobId = ? - ` - const { changes } = this.db.run(deleteSQL, [jobId]) - return changes === 1 - } - - // eslint-disable-next-line require-await - async createTable() { - /* although we have field called expireTimestamp, we are actually storing maxJobDuration in it */ - const createTableSQL = ` - CREATE TABLE IF NOT EXISTS ${this.schema.name} ( - owner TEXT, - did TEXT DEFAULT NULL, - jobId TEXT PRIMARY KEY, - dateCreated TEXT, - dateFinished TEXT DEFAULT NULL, - status INTEGER, - statusText TEXT, - results BLOB, - inputDID TEXT DEFAULT NULL, - algoDID TEXT DEFAULT NULL, - agreementId TEXT DEFAULT NULL, - expireTimestamp INTEGER, - environment TEXT DEFAULT NULL, - body BLOB - ); - ` - try { - this.db.exec(createTableSQL) - } catch (err) { - DATABASE_LOGGER.error(`Could not create ${this.schema.name} table: ${err.message}`) - throw err - } - } - - // eslint-disable-next-line require-await - async createImageTable(): Promise { - const createTableSQL = ` - CREATE TABLE IF NOT EXISTS docker_images ( - image TEXT PRIMARY KEY, - lastUsedTimestamp INTEGER NOT NULL - ); - ` - try { - this.db.exec(createTableSQL) - } catch (err) { - DATABASE_LOGGER.error('Could not create docker_images table: ' + err.message) - throw err - } - } - - // ── Service-on-Demand jobs ────────────────────────────────────────── - - // Cross-process lifecycle locks for service jobs. A row = an exclusive start/stop/ - // restart operation in flight on serviceId, held by `holder` (a per-process id). Rows - // are heartbeated (acquiredAt refreshed) while the operation runs; a row whose - // acquiredAt is older than the staleness window is a crashed holder and may be stolen. - // This extends the engine's in-memory serviceOpsInFlight guarantee to setups where - // several node processes share the same DB file + Docker daemon (e.g. a stale - // container still running during a redeploy) — in-memory sets cannot see each other. - // eslint-disable-next-line require-await - async createServiceLocksTable(): Promise { - const createTableSQL = ` - CREATE TABLE IF NOT EXISTS service_locks ( - serviceId TEXT PRIMARY KEY, - holder TEXT NOT NULL, - acquiredAt INTEGER NOT NULL - ); - ` - try { - this.db.exec(createTableSQL) - } catch (err) { - DATABASE_LOGGER.error('Could not create service_locks table: ' + err.message) - throw err - } - } - - // Atomically takes the lock for serviceId: inserts a fresh row, or steals one whose - // acquiredAt is older than staleMs (crashed holder). The single upsert statement is - // the atomicity guarantee — two processes racing it can never both see success. - // eslint-disable-next-line require-await - async acquireServiceLock( - serviceId: string, - holder: string, - staleMs: number - ): Promise { - const now = Date.now() - const upsertSQL = ` - INSERT INTO service_locks (serviceId, holder, acquiredAt) VALUES (?, ?, ?) - ON CONFLICT(serviceId) DO UPDATE - SET holder = excluded.holder, acquiredAt = excluded.acquiredAt - WHERE service_locks.acquiredAt <= ?; - ` - try { - const { changes } = this.db.run(upsertSQL, [serviceId, holder, now, now - staleMs]) - return changes === 1 - } catch (err) { - DATABASE_LOGGER.error(`Could not acquire service lock: ${err.message}`) - throw err - } - } - - // Releases only a lock we still hold — a stale lock stolen by another process must - // not be deleted out from under its new holder. - // eslint-disable-next-line require-await - async releaseServiceLock(serviceId: string, holder: string): Promise { - const deleteSQL = `DELETE FROM service_locks WHERE serviceId = ? AND holder = ?;` - try { - this.db.run(deleteSQL, [serviceId, holder]) - } catch (err) { - DATABASE_LOGGER.error(`Could not release service lock: ${err.message}`) - throw err - } - } - - // Heartbeat: re-stamps every lock this holder owns so long operations (multi-minute - // image pulls/builds) are not stolen as stale. - // eslint-disable-next-line require-await - async refreshServiceLocks(holder: string): Promise { - const updateSQL = `UPDATE service_locks SET acquiredAt = ? WHERE holder = ?;` - try { - this.db.run(updateSQL, [Date.now(), holder]) - } catch (err) { - DATABASE_LOGGER.error(`Could not refresh service locks: ${err.message}`) - throw err - } - } - - // True while ANY process holds a fresh lock on serviceId — used by read-only - // observers (e.g. the container health check) to avoid judging a service that - // another process is mid-way through restarting. - // eslint-disable-next-line require-await - async isServiceLocked(serviceId: string, staleMs: number): Promise { - const selectSQL = `SELECT acquiredAt FROM service_locks WHERE serviceId = ?;` - try { - const row = this.db.get<{ acquiredAt: number }>(selectSQL, [serviceId]) - return !!row && row.acquiredAt > Date.now() - staleMs - } catch (err) { - DATABASE_LOGGER.error(`Could not read service lock: ${err.message}`) - throw err - } - } - - // eslint-disable-next-line require-await - async createServiceTable(): Promise { - const createTableSQL = ` - CREATE TABLE IF NOT EXISTS service_jobs ( - serviceId TEXT PRIMARY KEY, - owner TEXT, - clusterHash TEXT, - status INTEGER, - expiresAt INTEGER, - dateCreated TEXT, - body BLOB - ); - ` - try { - this.db.exec(createTableSQL) - } catch (err) { - DATABASE_LOGGER.error('Could not create service_jobs table: ' + err.message) - throw err - } - } - - // eslint-disable-next-line require-await - async newServiceJob(job: ServiceJob): Promise { - const insertSQL = ` - INSERT INTO service_jobs - (serviceId, owner, clusterHash, status, expiresAt, dateCreated, body) - VALUES (?, ?, ?, ?, ?, ?, ?); - ` - try { - this.db.run(insertSQL, [ - job.serviceId, - job.owner, - job.clusterHash, - job.status, - job.expiresAt, - job.dateCreated, - Buffer.from(JSON.stringify({ ...job, updatedAt: Date.now() })) - ]) - } catch (err) { - DATABASE_LOGGER.error( - `Could not insert service job ${job.serviceId} on DB: ${err.message}` - ) - throw err - } - } - - // eslint-disable-next-line require-await - async updateServiceJob(job: ServiceJob): Promise { - const updateSQL = ` - UPDATE service_jobs - SET owner = ?, clusterHash = ?, status = ?, expiresAt = ?, body = ? - WHERE serviceId = ?; - ` - try { - const { changes } = this.db.run(updateSQL, [ - job.owner, - job.clusterHash, - job.status, - job.expiresAt, - Buffer.from(JSON.stringify({ ...job, updatedAt: Date.now() })), - job.serviceId - ]) - return changes - } catch (err) { - DATABASE_LOGGER.error( - `Error while updating service job ${job.serviceId}: ${err.message}` - ) - throw err - } - } - - // Persists ONE best-effort, non-lifecycle field onto a service job (runtime metrics, readiness) - // without clobbering lifecycle fields another process may have changed. The read + validate + - // merge + write run inside a single `BEGIN IMMEDIATE` transaction so the sequence is atomic even - // across processes sharing the SQLite file: IMMEDIATE takes the write lock up front (waiting up - // to busy_timeout), so no other process can commit a lifecycle change (status, expiry, container) - // between our read and our write. Inside the transaction it re-reads the CURRENT row, re-validates - // owner/clusterHash/status/containerId, applies `patch` to that current body, and writes back ONLY - // the `body` column guarded on the unchanged status — leaving every lifecycle field (incl. - // expiresAt, in both column and body) exactly as last committed. Returns true when a row was - // written; any mismatch/error rolls back and returns false. - private patchServiceJobBody( - serviceId: string, - expected: { - owner: string - clusterHash: string - status: number - containerId: string - }, - what: string, - patch: (body: ServiceJob) => void - ): boolean { - try { - this.db.exec('BEGIN IMMEDIATE;') - } catch (err) { - DATABASE_LOGGER.error( - `${what} update for service job ${serviceId}: could not begin transaction: ${err.message}` - ) - return false - } - try { - const row = this.db.get<{ body: Uint8Array }>( - `SELECT body FROM service_jobs WHERE serviceId = ?;`, - [serviceId] - ) - if (!row) { - this.db.exec('ROLLBACK;') - return false - } - const body = JSON.parse(Buffer.from(row.body).toString()) as ServiceJob - if ( - body.owner !== expected.owner || - body.clusterHash !== expected.clusterHash || - body.status !== expected.status || - body.containerId !== expected.containerId - ) { - this.db.exec('ROLLBACK;') - return false - } - patch(body) - const { changes } = this.db.run( - `UPDATE service_jobs SET body = ? WHERE serviceId = ? AND status = ?;`, - [ - Buffer.from(JSON.stringify({ ...body, updatedAt: Date.now() })), - serviceId, - expected.status - ] - ) - this.db.exec('COMMIT;') - return changes > 0 - } catch (err) { - DATABASE_LOGGER.error( - `Error while updating ${what} of service job ${serviceId}: ${err.message}` - ) - try { - this.db.exec('ROLLBACK;') - } catch { - // best-effort: transaction may already be closed - } - return false - } - } - - // eslint-disable-next-line require-await - async updateServiceJobMetrics( - serviceId: string, - expected: { - owner: string - clusterHash: string - status: number - containerId: string - }, - runtimeMetrics: ContainerMetricsSnapshot - ): Promise { - return this.patchServiceJobBody(serviceId, expected, 'metrics', (body) => { - body.runtimeMetrics = runtimeMetrics - }) - } - - // Same guarantees as updateServiceJobMetrics, for the readiness probe result: it is sampled from - // the same lease-free background loop, so it must never overwrite a lifecycle transition either. - // `readiness` is undefined for a service the node cannot probe. - // eslint-disable-next-line require-await - async updateServiceJobReadiness( - serviceId: string, - expected: { - owner: string - clusterHash: string - status: number - containerId: string - }, - readiness: ServiceReadiness | undefined, - modelDownload?: ServiceModelDownload - ): Promise { - return this.patchServiceJobBody(serviceId, expected, 'readiness', (body) => { - if (readiness) { - body.readiness = readiness - } - // Only overwritten when a fresh sample was taken: once the engine is ready the walk stops, - // and the last figures stay as the record of what was downloaded. - if (modelDownload) body.modelDownload = modelDownload - }) - } - - private mapServiceRows(rows: any[] | undefined): ServiceJob[] { - if (!rows || rows.length === 0) return [] - // BLOB comes back as Uint8Array from node:sqlite; decode through Buffer before parsing. - return rows.map((row) => JSON.parse(Buffer.from(row.body).toString()) as ServiceJob) - } - - // eslint-disable-next-line require-await - async getServiceJob(serviceId?: string, owner?: string): Promise { - const params: any[] = [] - let selectSQL = `SELECT * FROM service_jobs WHERE 1=1` - if (serviceId) { - selectSQL += ` AND serviceId = ?` - params.push(serviceId) - } - if (owner) { - // COLLATE NOCASE: callers are checksum-normalized on ingress (utils/evmAddress.ts), but - // rows written before that may hold another casing of the same address — and a missed - // match here reads as "no such service" rather than an error. - selectSQL += ` AND owner = ? COLLATE NOCASE` - params.push(owner) - } - try { - const rows = this.db.all(selectSQL, params) - return this.mapServiceRows(rows) - } catch (err) { - DATABASE_LOGGER.error(`getServiceJob query failed: ${err.message}`) - throw err - } - } - - // eslint-disable-next-line require-await - async getRunningServiceJobs(clusterHash?: string): Promise { - // Every status before Expired is "active": the consumer paid for the resources for a - // TIME WINDOW, so the reservation holds from the moment the record is created - // (Starting) through the whole start pipeline (Locking, image, Claiming), while - // Running/Restarting/Stopping, through Error (container died on its own) AND through - // an explicit Stopped — a stopped service can be restarted anytime on the same - // resources until expiresAt. Only expiry releases the - // reservation; nothing else may free it inside the paid window. - const activeStatuses = [ - ServiceStatusNumber.Starting, - ServiceStatusNumber.Locking, - ServiceStatusNumber.PullImage, - ServiceStatusNumber.BuildImage, - ServiceStatusNumber.Claiming, - ServiceStatusNumber.Running, - ServiceStatusNumber.Restarting, - ServiceStatusNumber.Stopping, - ...SERVICE_STOPPED_STATUSES, - ServiceStatusNumber.Error - ] - const placeholders = activeStatuses.map(() => '?').join(',') - const params: Array = [...activeStatuses] - let selectSQL = `SELECT * FROM service_jobs WHERE status IN (${placeholders})` - if (clusterHash) { - selectSQL += ` AND clusterHash = ?` - params.push(clusterHash) - } - try { - const rows = this.db.all(selectSQL, params) - // The reservation is tied to PAYMENT: an Error/Stopped job whose payment was - // never claimed (escrow lock failed — e.g. insufficient funds — or refunded) - // must not hold resources, or anyone could squat a node's GPU for free by - // starting services against an empty escrow account. Mid-pipeline statuses - // keep reserving even without claimTx — they are en route to payment. - // JS-side filter because payment lives in the JSON body, not a SQL column. - return this.mapServiceRows(rows).filter( - (j) => - (j.status !== ServiceStatusNumber.Error && !isStoppedLike(j.status)) || - !!j.payment?.claimTx - ) - } catch (err) { - DATABASE_LOGGER.error(`getRunningServiceJobs query failed: ${err.message}`) - throw err - } - } - - // Expired (terminal) services whose paid window ended at or before `expiresBefore` (Unix - // ms) and that still hold output archives — candidates for deleting them once storage - // expiry has elapsed. The archive filter runs in SQL: expired rows are kept forever (with - // outputArchives: [] once cleaned), so without it every cleanup run would load and parse - // every service that ever expired. - // eslint-disable-next-line require-await - async getExpiredServiceJobsBefore( - expiresBefore: number, - clusterHash?: string - ): Promise { - const params: Array = [ServiceStatusNumber.Expired, expiresBefore] - let selectSQL = - `SELECT * FROM service_jobs WHERE status = ? AND expiresAt <= ?` + - ` AND json_array_length(CAST(body AS TEXT), '$.outputArchives') > 0` - if (clusterHash) { - selectSQL += ` AND clusterHash = ?` - params.push(clusterHash) - } - try { - const rows = this.db.all(selectSQL, params) - return this.mapServiceRows(rows) - } catch (err) { - DATABASE_LOGGER.error(err.message) - throw err - } - } - - // eslint-disable-next-line require-await - async getExpiredServiceJobs(clusterHash?: string): Promise { - // Running, Error AND Stopped all still hold their paid reservation (see activeStatuses - // above), so all three must be swept once past expiresAt: the sweep is the ONLY place - // the reservation is released. Without it an abandoned Error or Stopped service would - // keep its resources/ports forever and still read as restartable. Stopping is swept - // too: a process crash mid-stop persists that status and nothing else recovers it - // (it is not a pending-start status), so it would otherwise be stuck reserving - // resources forever — the sweep's doStopService handles a Stopping row like any - // other teardown (benign 404s for whatever the crashed stop already removed). - const expirableStatuses = [ - ServiceStatusNumber.Running, - ServiceStatusNumber.Error, - ...SERVICE_STOPPED_STATUSES, - ServiceStatusNumber.Stopping - ] - const placeholders = expirableStatuses.map(() => '?').join(',') - const params: Array = [...expirableStatuses, Date.now()] - let selectSQL = `SELECT * FROM service_jobs WHERE status IN (${placeholders}) AND expiresAt <= ?` - if (clusterHash) { - selectSQL += ` AND clusterHash = ?` - params.push(clusterHash) - } - try { - const rows = this.db.all(selectSQL, params) - return this.mapServiceRows(rows) - } catch (err) { - DATABASE_LOGGER.error(`getExpiredServiceJobs query failed: ${err.message}`) - throw err - } - } - - // Service jobs that are mid-start and need the background loop to advance them. - // Starting = fresh (handler just created it); the intermediate states are picked up too so - // the loop can resume / orphan-recover them after a node restart. - // eslint-disable-next-line require-await - async getPendingServiceStarts(clusterHash?: string): Promise { - const startStatuses = SERVICE_START_PENDING_STATUSES - const placeholders = startStatuses.map(() => '?').join(',') - const params: Array = [...startStatuses] - let selectSQL = `SELECT * FROM service_jobs WHERE status IN (${placeholders})` - if (clusterHash) { - selectSQL += ` AND clusterHash = ?` - params.push(clusterHash) - } - try { - const rows = this.db.all(selectSQL, params) - return this.mapServiceRows(rows) - } catch (err) { - DATABASE_LOGGER.error(`getPendingServiceStarts query failed: ${err.message}`) - throw err - } - } - - // eslint-disable-next-line require-await - async updateImage(image: string): Promise { - const timestamp = Math.floor(Date.now() / 1000) // Unix timestamp in seconds - const insertSQL = ` - INSERT OR REPLACE INTO docker_images (image, lastUsedTimestamp) - VALUES (?, ?); - ` - try { - this.db.run(insertSQL, [image, timestamp]) - DATABASE_LOGGER.debug(`Updated image usage timestamp for ${image}`) - } catch (err) { - DATABASE_LOGGER.error(`Could not update image usage for ${image}: ${err.message}`) - throw err - } - } - - // eslint-disable-next-line require-await - async deleteImage(image: string): Promise { - const deleteSQL = ` - DELETE FROM docker_images WHERE image = ?; - ` - try { - this.db.run(deleteSQL, [image]) - DATABASE_LOGGER.debug(`Deleted image ${image}`) - } catch (err) { - DATABASE_LOGGER.error(`Could not delete image ${image}: ${err.message}`) - throw err - } - } - - // eslint-disable-next-line require-await - async getOldImages(retentionDays: number = 7): Promise { - const cutoffTimestamp = Math.floor(Date.now() / 1000) - retentionDays * 24 * 60 * 60 - const selectSQL = ` - SELECT image FROM docker_images - WHERE lastUsedTimestamp < ? - ORDER BY lastUsedTimestamp ASC; - ` - try { - const rows = this.db.all<{ image: string }>(selectSQL, [cutoffTimestamp]) - return rows.map((row) => row.image) - } catch (err) { - DATABASE_LOGGER.error(`Could not get old images: ${err.message}`) - throw err - } - } - - // eslint-disable-next-line require-await - async newJob(job: DBComputeJob): Promise { - // TO DO C2D - const insertSQL = ` - INSERT INTO ${this.schema.name} - ( - owner, - did, - jobId, - dateCreated, - status, - statusText, - inputDID, - algoDID, - agreementId, - expireTimestamp, - environment, - body - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); - ` - - try { - this.db.run(insertSQL, [ - job.owner, - job.did, - job.jobId, - job.dateCreated || String(Date.now() / 1000), // seconds from epoch, - job.status || C2DStatusNumber.JobStarted, - job.statusText || C2DStatusText.JobStarted, - job.inputDID ? convertArrayToString(job.inputDID) : job.inputDID, - job.algoDID, - job.agreementId, - job.maxJobDuration, - job.environment, - generateBlobFromJSON(job) - ]) - DATABASE_LOGGER.debug( - `C2D job ${job.clusterHash}-${job.jobId} created (env ${job.environment})` - ) - return job.jobId - } catch (err) { - DATABASE_LOGGER.error(`Could not insert C2D job ${job.jobId} on DB: ${err.message}`) - throw err - } - } - - /** - * on a get status for instance, all params are optional - * but at least one is required... In case we don't have a jobId, - * we have multiple results (by owner for instance) - * So, it refines the query or we can have more than 1 result (same as current implementation) - * @param jobId the job identifier - * @param agreementId the agreement identifier (did ?) - * @param owner the consumer address / job owner - * @returns job(s) - */ - // eslint-disable-next-line require-await - async getJob( - jobId?: string, - agreementId?: string, - owner?: string - ): Promise { - const params: any = [] - let selectSQL = `SELECT * FROM ${this.schema.name} WHERE 1=1` - if (jobId) { - selectSQL += ` AND jobId = ?` - params.push(jobId) - } - if (agreementId) { - if (!agreementId.startsWith('0x')) { - agreementId = '0x' + agreementId - } - selectSQL += ` AND agreementId = ?` - params.push(agreementId) - } - if (owner) { - // COLLATE NOCASE — see getServiceJob: an owner casing mismatch must not read as - // "job not found". - selectSQL += ` AND owner = ? COLLATE NOCASE` - params.push(owner) - } - - let rows: any[] - try { - rows = this.db.all(selectSQL, params) - } catch (err) { - DATABASE_LOGGER.error(`getJob query failed: ${err.message}`) - throw err - } - // also decode the internal data into job data - if (rows && rows.length > 0) { - const all: DBComputeJob[] = rows.map((row) => { - const body = generateJSONFromBlob(row.body) - delete row.body - const maxJobDuration = row.expireTimestamp - delete row.expireTimestamp - const job: DBComputeJob = { ...row, ...body, maxJobDuration } - return job - }) - return all - } - DATABASE_LOGGER.debug( - `Could not find any job with jobId: ${jobId}, agreementId: ${agreementId}, or owner: ${owner} in database!` - ) - return [] - } - - // eslint-disable-next-line require-await - async updateJob(job: DBComputeJob): Promise { - // if (job.dateFinished && job.isRunning) { - // job.isRunning = false - // } - // TO DO C2D - const data: any[] = [ - job.owner, - job.status, - job.statusText, - job.maxJobDuration, - generateBlobFromJSON(job), - job.dateFinished, - job.jobId - ] - const updateSQL = ` - UPDATE ${this.schema.name} - SET - owner = ?, - status = ?, - statusText = ?, - expireTimestamp = ?, - body = ?, - dateFinished = ? - WHERE jobId = ?; - ` - - try { - // number of rows updated successfully - const { changes } = this.db.run(updateSQL, data) - return changes - } catch (err) { - DATABASE_LOGGER.error(`Error while updating job ${job.jobId}: ${err.message}`) - throw err - } - } - - // eslint-disable-next-line require-await - async getRunningJobs(engine?: string, environment?: string): Promise { - const selectSQL = ` - SELECT * FROM ${this.schema.name} WHERE dateFinished IS NULL ORDER by dateCreated - ` - let rows: any[] - try { - rows = this.db.all(selectSQL) - } catch (err) { - DATABASE_LOGGER.error(`getRunningJobs query failed: ${err.message}`) - throw err - } - // also decode the internal data into job data - // get them all running - if (rows && rows.length > 0) { - const all: DBComputeJob[] = rows.map((row) => { - const body = generateJSONFromBlob(row.body) - delete row.body - const maxJobDuration = row.expireTimestamp - delete row.expireTimestamp - const job: DBComputeJob = { ...row, ...body, maxJobDuration } - return job - }) - // filter them out - const filtered = all.filter((job) => { - let include = true - if (engine && engine !== job.clusterHash) { - include = false - } - if (environment && environment !== job.environment) { - include = false - } - if (job.dateFinished) { - include = false - } - return include - }) - return filtered - } - // DATABASE_LOGGER.info('Could not find any running C2D jobs!') - return [] - } - - // eslint-disable-next-line require-await - async getFinishedJobs(environments?: string[]): Promise { - let selectSQL = ` - SELECT * FROM ${this.schema.name} WHERE (dateFinished IS NOT NULL OR results IS NOT NULL) - ` - const params: string[] = [] - if (environments && environments.length > 0) { - const placeholders = environments.map(() => '?').join(',') - selectSQL += ` AND environment IN (${placeholders})` - params.push(...environments) - } - - selectSQL += ` ORDER BY dateFinished DESC` - - let rows: any[] - try { - rows = this.db.all(selectSQL, params) - } catch (err) { - DATABASE_LOGGER.error(`getFinishedJobs query failed: ${err.message}`) - throw err - } - // also decode the internal data into job data - // get them all running - if (rows && rows.length > 0) { - const all: DBComputeJob[] = rows.map((row) => { - const body = generateJSONFromBlob(row.body) - delete row.body - const maxJobDuration = row.expireTimestamp - delete row.expireTimestamp - const job: DBComputeJob = { ...row, ...body, maxJobDuration } - return job - }) - return all - } - environments - ? DATABASE_LOGGER.debug( - 'No jobs found for the specified enviroments: ' + environments.join(',') - ) - : DATABASE_LOGGER.debug('No jobs found') - return [] - } - - /** - * @param fromTimestamp lower bound in Unix **seconds** (the unit this table's - * dateCreated/dateFinished TEXT columns store). Callers must normalize before getting - * here — see parseFromTimestampSeconds in core/utils/timestamps.ts. Taking a number - * rather than a string is deliberate: a raw string bound to these columns is compared - * with memcmp, which happens to work for 10-digit seconds and silently matches nothing - * for milliseconds, ISO dates or garbage. - */ - async getJobs( - environments?: string[], - fromTimestamp?: number, - consumerAddrs?: string[], - status?: C2DStatusNumber, - runningJobs?: boolean - ): Promise { - let selectSQL = `SELECT * FROM ${this.schema.name}` - - const params: Array = [] - const conditions: string[] = [] - - if (environments && environments.length > 0) { - const placeholders = environments.map(() => '?').join(',') - conditions.push(`environment IN (${placeholders})`) - params.push(...environments) - } - - // dateCreated/dateFinished are TEXT holding decimal seconds, so they must be CAST for - // the comparison to be numeric instead of lexicographic. - if (runningJobs) { - conditions.push(`status = ?`) - params.push(C2DStatusNumber.RunningAlgorithm.toString()) - if (fromTimestamp !== undefined && fromTimestamp !== null) { - conditions.push(`CAST(dateCreated AS REAL) >= ?`) - params.push(fromTimestamp) - } - } else { - if (fromTimestamp !== undefined && fromTimestamp !== null) { - conditions.push(`CAST(dateFinished AS REAL) >= ?`) - params.push(fromTimestamp) - } - // C2DStatusNumber.JobStarted is 0, so a truthiness check would silently drop the - // filter and return every status instead. - if (status !== undefined && status !== null) { - conditions.push(`status = ?`) - params.push(status.toString()) - } - } - - if (consumerAddrs && consumerAddrs.length > 0) { - const placeholders = consumerAddrs.map(() => '?').join(',') - // COLLATE NOCASE for the same reason as getJob/getServiceJob: match an owner whatever - // casing the row was written with. - conditions.push(`owner COLLATE NOCASE IN (${placeholders})`) - params.push(...consumerAddrs) - } - - if (conditions.length > 0) { - selectSQL += ` WHERE ${conditions.join(' AND ')}` - } - // Numeric ordering for the same reason as the comparison above. NOTE: do NOT copy this - // CAST to the service_jobs table — its dateCreated is an ISO string, which sorts - // correctly as text and would CAST to its year. - selectSQL += ` ORDER BY CAST(dateCreated AS REAL) DESC` - return await this.doQuery(selectSQL, params, environments) - } - - async getJobsByStatus( - environments: string[], - status: C2DStatusNumber[] - ): Promise { - let selectSQL = `SELECT * FROM ${this.schema.name}` - - // node:sqlite bindings accept both strings and numbers; `status` is a numeric enum. - const params: Array = [] - const conditions: string[] = [] - - if (environments && environments.length > 0) { - const placeholders = environments.map(() => '?').join(',') - conditions.push(`environment IN (${placeholders})`) - params.push(...environments) - } - - if (status && status.length > 0) { - const placeholders = status.map(() => '?').join(',') - conditions.push(`status IN (${placeholders})`) - params.push(...status) - } - - if (conditions.length > 0) { - selectSQL += ` WHERE ${conditions.join(' AND ')}` - } - selectSQL += ` ORDER BY dateCreated DESC` - - return await this.doQuery(selectSQL, params, environments) - } - - // eslint-disable-next-line require-await - private async doQuery( - selectSQL: string, - params: Array, - environments: string[] - ): Promise { - let rows: any[] - try { - rows = this.db.all(selectSQL, params) - } catch (err) { - DATABASE_LOGGER.error(`C2D jobs query failed: ${err.message}`) - throw err - } - // also decode the internal data into job data - // get them all running - if (rows && rows.length > 0) { - const all: DBComputeJob[] = rows.map((row) => { - const body = generateJSONFromBlob(row.body) - delete row.body - const maxJobDuration = row.expireTimestamp - delete row.expireTimestamp - const job: DBComputeJob = { ...row, ...body, maxJobDuration } - if (!job.jobIdHash && job.jobId) { - job.jobIdHash = create256Hash(job.jobId) - } - return job - }) - return all - } - environments - ? DATABASE_LOGGER.debug( - 'No jobs found for the specified enviroments: ' + environments.join(',') - ) - : DATABASE_LOGGER.debug('No jobs found') - return [] - } -} diff --git a/src/components/database/sqliteNodeMetrics.ts b/src/components/database/sqliteNodeMetrics.ts deleted file mode 100644 index 3feaae086..000000000 --- a/src/components/database/sqliteNodeMetrics.ts +++ /dev/null @@ -1,441 +0,0 @@ -import { SqliteClient } from './sqliteClient.js' -import { DATABASE_LOGGER } from '../../utils/logging/common.js' -import type { - NodeMetricsEnvResource, - NodeMetricsGpu, - NodeMetricsHourly, - NodeMetricsSnapshot -} from '../../@types/nodeMetrics.js' - -const HOUR_MS = 60 * 60 * 1000 - -export function floorToHour(epochMs: number): number { - return Math.floor(epochMs / HOUR_MS) * HOUR_MS -} - -// Raw sample row as stored (scalars flattened, structured fields as JSON strings). -interface SampleRow { - collectedAt: number - cpuUsagePercent: number - coresAllocated: number - hostCores: number - throttledCount: number - memUsedBytes: number - memLimitBytes: number - hostFreeBytes: number - hostTotalBytes: number - diskUsedBytes: number - rxBytes: number - txBytes: number - jobsRunning: number - jobsRunningFree: number - jobsQueued: number - jobsQueuedFree: number - sampledContainers: number - gpu: string - env: string -} - -// Hourly row as stored. -interface HourlyRow extends Omit { - hourStart: number - sampleCount: number -} - -function safeParse(json: string | null | undefined, fallback: T): T { - if (!json) return fallback - try { - return JSON.parse(json) as T - } catch { - return fallback - } -} - -/** - * SQLite-backed history layer for per-node resource metrics. - * - * Two tables: - * - `node_metrics_samples` — raw per-minute snapshots (short-lived buffer, purged after a few - * hours). Input to the hourly averaging. - * - `node_metrics_hourly` — one row per floored UTC hour, averaged scalars + `sampleCount`, - * GPU/env averaged and stored as JSON. This is what `getNodeMetricsHistory` reads. - * - * Uses the shared synchronous `SqliteClient`; every table is small and bounded (minute buffer - * plus ~6 months of hourly rows), so main-thread queries are acceptable, matching the other - * embedded SQLite databases (nonce / c2d / auth). - */ -export class NodeMetricsDatabase { - private db: SqliteClient - - constructor(dbFilePath: string = 'databases/nodeMetrics.sqlite') { - // SqliteClient creates the parent directory on construction. - this.db = new SqliteClient(dbFilePath) - this.createTables() - DATABASE_LOGGER.debug('Node metrics Database initiated with SQLite provider') - } - - private createTables(): void { - this.db.exec(` - CREATE TABLE IF NOT EXISTS node_metrics_samples ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - collectedAt INTEGER NOT NULL, - cpuUsagePercent REAL DEFAULT 0, - coresAllocated REAL DEFAULT 0, - hostCores REAL DEFAULT 0, - throttledCount REAL DEFAULT 0, - memUsedBytes REAL DEFAULT 0, - memLimitBytes REAL DEFAULT 0, - hostFreeBytes REAL DEFAULT 0, - hostTotalBytes REAL DEFAULT 0, - diskUsedBytes REAL DEFAULT 0, - rxBytes REAL DEFAULT 0, - txBytes REAL DEFAULT 0, - jobsRunning REAL DEFAULT 0, - jobsRunningFree REAL DEFAULT 0, - jobsQueued REAL DEFAULT 0, - jobsQueuedFree REAL DEFAULT 0, - sampledContainers REAL DEFAULT 0, - gpu TEXT DEFAULT '[]', - env TEXT DEFAULT '[]' - ) - `) - this.db.exec( - `CREATE INDEX IF NOT EXISTS idx_node_metrics_samples_collectedAt ON node_metrics_samples (collectedAt)` - ) - this.db.exec(` - CREATE TABLE IF NOT EXISTS node_metrics_hourly ( - hourStart INTEGER PRIMARY KEY, - sampleCount INTEGER DEFAULT 0, - cpuUsagePercent REAL DEFAULT 0, - coresAllocated REAL DEFAULT 0, - hostCores REAL DEFAULT 0, - throttledCount REAL DEFAULT 0, - memUsedBytes REAL DEFAULT 0, - memLimitBytes REAL DEFAULT 0, - hostFreeBytes REAL DEFAULT 0, - hostTotalBytes REAL DEFAULT 0, - diskUsedBytes REAL DEFAULT 0, - rxBytes REAL DEFAULT 0, - txBytes REAL DEFAULT 0, - jobsRunning REAL DEFAULT 0, - jobsRunningFree REAL DEFAULT 0, - jobsQueued REAL DEFAULT 0, - jobsQueuedFree REAL DEFAULT 0, - sampledContainers REAL DEFAULT 0, - gpu TEXT DEFAULT '[]', - env TEXT DEFAULT '[]' - ) - `) - this.db.exec( - `CREATE INDEX IF NOT EXISTS idx_node_metrics_hourly_hourStart ON node_metrics_hourly (hourStart)` - ) - } - - insertSample(snapshot: NodeMetricsSnapshot): void { - this.db.run( - `INSERT INTO node_metrics_samples ( - collectedAt, cpuUsagePercent, coresAllocated, hostCores, throttledCount, - memUsedBytes, memLimitBytes, hostFreeBytes, hostTotalBytes, diskUsedBytes, - rxBytes, txBytes, jobsRunning, jobsRunningFree, jobsQueued, jobsQueuedFree, - sampledContainers, gpu, env - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - [ - snapshot.collectedAt, - snapshot.cpu.usagePercent, - snapshot.cpu.coresAllocated, - snapshot.cpu.hostCores, - snapshot.cpu.throttledCount, - snapshot.memory.usedBytes, - snapshot.memory.limitBytes, - snapshot.memory.hostFreeBytes, - snapshot.memory.hostTotalBytes, - snapshot.disk.usedBytes, - snapshot.network.rxBytes, - snapshot.network.txBytes, - snapshot.jobs.running, - snapshot.jobs.runningFree, - snapshot.jobs.queued, - snapshot.jobs.queuedFree, - snapshot.meta.sampledContainers, - JSON.stringify(snapshot.gpu ?? []), - JSON.stringify(snapshot.env ?? []) - ] - ) - } - - // Distinct floored hours that have raw samples strictly before `beforeHour` (i.e. complete - // hours ready to roll up). `beforeHour` is a floored-hour epoch ms. - getPendingSampleHours(beforeHour: number): number[] { - // HOUR_MS is a compile-time module constant, not user input — the interpolation below is - // not an injection surface. Every user-supplied value is bound via the params array. - const rows = this.db.all<{ hourStart: number }>( - `SELECT DISTINCT (collectedAt / ${HOUR_MS}) * ${HOUR_MS} AS hourStart - FROM node_metrics_samples - WHERE collectedAt < ? - ORDER BY hourStart ASC`, - [beforeHour] - ) - return rows.map((r) => Math.floor(Number(r.hourStart))) - } - - // Average every raw sample in [hourStart, hourStart+1h) into one hourly row, upsert it, then - // delete the consumed raw samples. No samples → no row written. - rollupHour(hourStart: number): boolean { - const from = hourStart - const to = hourStart + HOUR_MS - const samples = this.db.all( - `SELECT * FROM node_metrics_samples WHERE collectedAt >= ? AND collectedAt < ?`, - [from, to] - ) - if (samples.length === 0) return false - - const agg = this.aggregateRows(samples, hourStart) - - this.db.run( - `INSERT INTO node_metrics_hourly ( - hourStart, sampleCount, cpuUsagePercent, coresAllocated, hostCores, throttledCount, - memUsedBytes, memLimitBytes, hostFreeBytes, hostTotalBytes, diskUsedBytes, - rxBytes, txBytes, jobsRunning, jobsRunningFree, jobsQueued, jobsQueuedFree, - sampledContainers, gpu, env - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(hourStart) DO UPDATE SET - sampleCount = excluded.sampleCount, - cpuUsagePercent = excluded.cpuUsagePercent, - coresAllocated = excluded.coresAllocated, - hostCores = excluded.hostCores, - throttledCount = excluded.throttledCount, - memUsedBytes = excluded.memUsedBytes, - memLimitBytes = excluded.memLimitBytes, - hostFreeBytes = excluded.hostFreeBytes, - hostTotalBytes = excluded.hostTotalBytes, - diskUsedBytes = excluded.diskUsedBytes, - rxBytes = excluded.rxBytes, - txBytes = excluded.txBytes, - jobsRunning = excluded.jobsRunning, - jobsRunningFree = excluded.jobsRunningFree, - jobsQueued = excluded.jobsQueued, - jobsQueuedFree = excluded.jobsQueuedFree, - sampledContainers = excluded.sampledContainers, - gpu = excluded.gpu, - env = excluded.env`, - hourlyToParams(agg) - ) - - this.db.run( - `DELETE FROM node_metrics_samples WHERE collectedAt >= ? AND collectedAt < ?`, - [from, to] - ) - return true - } - - // Live, unstored average of the raw samples in an as-yet-unrolled hour (the current hour) — the - // read-only counterpart to `rollupHour`, sharing the exact averaging via `aggregateRows`. Marks - // the result `partial: true`. Returns null when no samples exist for that hour yet. - getPartialHour(hourStart: number): NodeMetricsHourly | null { - const samples = this.db.all( - `SELECT * FROM node_metrics_samples WHERE collectedAt >= ? AND collectedAt < ?`, - [hourStart, hourStart + HOUR_MS] - ) - if (samples.length === 0) return null - const agg = this.aggregateRows(samples, hourStart) - agg.partial = true - return agg - } - - // Averages a set of raw sample rows into one NodeMetricsHourly bucket. Pure (no DB writes) so - // both the stored roll-up and the live partial-hour produce byte-for-byte identical shapes. - private aggregateRows(samples: SampleRow[], hourStart: number): NodeMetricsHourly { - const n = samples.length - const mean = (pick: (s: SampleRow) => number): number => - samples.reduce((acc, s) => acc + (Number(pick(s)) || 0), 0) / n - return { - hourStart, - sampleCount: n, - cpu: { - usagePercent: mean((s) => s.cpuUsagePercent), - coresAllocated: mean((s) => s.coresAllocated), - hostCores: mean((s) => s.hostCores), - throttledCount: mean((s) => s.throttledCount) - }, - memory: { - usedBytes: mean((s) => s.memUsedBytes), - limitBytes: mean((s) => s.memLimitBytes), - hostFreeBytes: mean((s) => s.hostFreeBytes), - hostTotalBytes: mean((s) => s.hostTotalBytes) - }, - disk: { usedBytes: mean((s) => s.diskUsedBytes) }, - network: { rxBytes: mean((s) => s.rxBytes), txBytes: mean((s) => s.txBytes) }, - jobs: { - running: mean((s) => s.jobsRunning), - runningFree: mean((s) => s.jobsRunningFree), - queued: mean((s) => s.jobsQueued), - queuedFree: mean((s) => s.jobsQueuedFree) - }, - gpu: averageGpu(samples.map((s) => safeParse(s.gpu, []))), - env: averageEnv(samples.map((s) => safeParse(s.env, []))), - meta: { sampledContainers: mean((s) => s.sampledContainers) } - } - } - - // Safety net: drop raw samples older than the buffer window (normal path consumes them in the - // hourly rollup). `olderThan` is an epoch ms cutoff. - purgeOldSamples(olderThan: number): number { - const res = this.db.run(`DELETE FROM node_metrics_samples WHERE collectedAt < ?`, [ - olderThan - ]) - return res.changes - } - - // Retention: drop hourly rows older than the cutoff. Returns rows deleted. - deleteHourlyOlderThan(cutoff: number): number { - const res = this.db.run(`DELETE FROM node_metrics_hourly WHERE hourStart < ?`, [ - cutoff - ]) - return res.changes - } - - // Ordered hourly buckets in [from, to], capped at `limit` rows. - getHourly(from: number, to: number, limit: number): NodeMetricsHourly[] { - const rows = this.db.all( - `SELECT * FROM node_metrics_hourly - WHERE hourStart >= ? AND hourStart <= ? - ORDER BY hourStart ASC - LIMIT ?`, - [from, to, limit] - ) - return rows.map((r) => hourlyRowToApi(r)) - } -} - -// Flattens a NodeMetricsHourly into the positional bind array for the node_metrics_hourly -// INSERT (column order must match the statement in rollupHour). -function hourlyToParams(h: NodeMetricsHourly): unknown[] { - return [ - h.hourStart, - h.sampleCount, - h.cpu.usagePercent, - h.cpu.coresAllocated, - h.cpu.hostCores, - h.cpu.throttledCount, - h.memory.usedBytes, - h.memory.limitBytes, - h.memory.hostFreeBytes, - h.memory.hostTotalBytes, - h.disk.usedBytes, - h.network.rxBytes, - h.network.txBytes, - h.jobs.running, - h.jobs.runningFree, - h.jobs.queued, - h.jobs.queuedFree, - h.meta.sampledContainers, - JSON.stringify(h.gpu), - JSON.stringify(h.env) - ] -} - -function hourlyRowToApi(r: HourlyRow): NodeMetricsHourly { - return { - hourStart: Number(r.hourStart), - sampleCount: Number(r.sampleCount), - cpu: { - usagePercent: Number(r.cpuUsagePercent), - coresAllocated: Number(r.coresAllocated), - hostCores: Number(r.hostCores), - throttledCount: Number(r.throttledCount) - }, - memory: { - usedBytes: Number(r.memUsedBytes), - limitBytes: Number(r.memLimitBytes), - hostFreeBytes: Number(r.hostFreeBytes), - hostTotalBytes: Number(r.hostTotalBytes) - }, - disk: { usedBytes: Number(r.diskUsedBytes) }, - network: { rxBytes: Number(r.rxBytes), txBytes: Number(r.txBytes) }, - jobs: { - running: Number(r.jobsRunning), - runningFree: Number(r.jobsRunningFree), - queued: Number(r.jobsQueued), - queuedFree: Number(r.jobsQueuedFree) - }, - gpu: safeParse(r.gpu, []), - env: safeParse(r.env, []), - meta: { sampledContainers: Number(r.sampledContainers) } - } -} - -// Average each numeric GPU field per resourceId across every sample that reported that device. -// A missing metric on a given sample simply does not contribute to that field's mean. -function averageGpu(perSample: NodeMetricsGpu[][]): NodeMetricsGpu[] { - const acc = new Map< - string, - { - vendor?: string - sums: Record - counts: Record - } - >() - const numericFields: (keyof NodeMetricsGpu)[] = [ - 'utilizationPercent', - 'memoryUsedBytes', - 'memoryTotalBytes', - 'temperatureC', - 'powerWatts' - ] - for (const list of perSample) { - for (const g of list ?? []) { - const id = String(g.resourceId) - if (!acc.has(id)) acc.set(id, { vendor: g.vendor, sums: {}, counts: {} }) - const entry = acc.get(id) - if (g.vendor && !entry.vendor) entry.vendor = g.vendor - for (const f of numericFields) { - const v = g[f] - if (typeof v === 'number' && Number.isFinite(v)) { - entry.sums[f] = (entry.sums[f] ?? 0) + v - entry.counts[f] = (entry.counts[f] ?? 0) + 1 - } - } - } - } - const out: NodeMetricsGpu[] = [] - for (const [resourceId, entry] of acc) { - const gpu: NodeMetricsGpu = { resourceId, vendor: entry.vendor } - for (const f of numericFields) { - if (entry.counts[f] > 0) { - ;(gpu[f] as number) = entry.sums[f] / entry.counts[f] - } - } - out.push(gpu) - } - return out -} - -// Average total/inUse per env+resource across every sample that reported that pair. -function averageEnv(perSample: NodeMetricsEnvResource[][]): NodeMetricsEnvResource[] { - const acc = new Map< - string, - { env: string; resource: string; total: number; inUse: number; count: number } - >() - for (const list of perSample) { - for (const e of list ?? []) { - const key = `${e.env} ${e.resource}` - if (!acc.has(key)) { - acc.set(key, { env: e.env, resource: e.resource, total: 0, inUse: 0, count: 0 }) - } - const entry = acc.get(key) - entry.total += Number(e.total) || 0 - entry.inUse += Number(e.inUse) || 0 - entry.count += 1 - } - } - const out: NodeMetricsEnvResource[] = [] - for (const entry of acc.values()) { - out.push({ - env: entry.env, - resource: entry.resource, - total: entry.count > 0 ? entry.total / entry.count : 0, - inUse: entry.count > 0 ? entry.inUse / entry.count : 0 - }) - } - return out -} diff --git a/src/components/database/typesense.ts b/src/components/database/typesense.ts deleted file mode 100644 index aa157a81c..000000000 --- a/src/components/database/typesense.ts +++ /dev/null @@ -1,204 +0,0 @@ -import { - TypesenseSearchParams, - TypesenseCollectionCreateSchema, - TypesenseCollectionSchema, - TypesenseCollectionUpdateSchema, - TypesenseConfigOptions, - TypesenseDocumentSchema, - TypesenseSearchResponse -} from '../../@types/index.js' -import { TYPESENSE_HITS_CAP } from '../../utils/constants.js' -import { TypesenseApi } from './typesenseApi.js' -import { TypesenseConfig } from './typesenseConfig.js' - -export const convertTypesenseConfig = (url: string) => { - const urlObject = new URL(url) - const apiKey = urlObject.searchParams.get('apiKey') - const config: TypesenseConfigOptions = { - apiKey, - nodes: [ - { - host: urlObject.hostname, - port: urlObject.port, - protocol: urlObject.protocol.split(':')[0] - } - ] - } - return config -} - -export class TypesenseError extends Error { - httpStatus?: number - - constructor(message?: string) { - super(message) - this.name = new.target.name - Object.setPrototypeOf(this, new.target.prototype) - } -} - -/** - * TypesenseDocuments class implements CRUD methods - * for interacting with documents of an individual collection - * In addition, it implements a method for searching in documents - */ -class TypesenseDocuments { - apiPath: string - - constructor( - private collectionName: string, - private api: TypesenseApi - ) { - this.apiPath = `/collections/${this.collectionName}/documents` - } - - // eslint-disable-next-line require-await - async create(document: TypesenseDocumentSchema) { - if (!document) throw new Error('No document provided') - return this.api.post(this.apiPath, document) - } - - // eslint-disable-next-line require-await - async upsert(document: TypesenseDocumentSchema) { - if (!document) throw new Error('No document provided') - return this.api.post(this.apiPath, document, { - action: 'upsert' - }) - } - - // eslint-disable-next-line require-await - async retrieve(documentId: string) { - const path = `${this.apiPath}/${documentId}` - return this.api.get(path) - } - - // eslint-disable-next-line require-await - async delete(documentId: string) { - const path = `${this.apiPath}/${documentId}` - return this.api.delete(path) - } - - // eslint-disable-next-line require-await - async deleteByChainId(filterCondition: string, batchSize: number = TYPESENSE_HITS_CAP) { - const batch = Math.min(batchSize, TYPESENSE_HITS_CAP) - return this.api.delete(this.apiPath, { - filter_by: filterCondition, - batch_size: batch - }) - } - - // eslint-disable-next-line require-await - async update(documentId: string, partialDocument: Partial) { - const path = `${this.apiPath}/${documentId}` - return this.api.patch(path, partialDocument) - } - - // eslint-disable-next-line require-await - async search( - searchParameters: TypesenseSearchParams - ): Promise { - const additionalQueryParams: { [key: string]: any } = {} - for (const key in searchParameters) { - if (Array.isArray(searchParameters[key])) { - additionalQueryParams[key] = searchParameters[key].join(',') - } - } - const queryParams = Object.assign({}, searchParameters, additionalQueryParams) - const path = `${this.apiPath}/search` - return this.api.get( - path, - queryParams - ) as Promise - } -} - -/** - * TypesenseCollection class implements CRUD methods for interacting with an individual collection - * It initiates class that provides access to methods of documents - */ -class TypesenseCollection { - apiPath: string - private readonly _documents: TypesenseDocuments - - constructor( - private name: string, - private api: TypesenseApi - ) { - this.apiPath = `/collections/${this.name}` - this._documents = new TypesenseDocuments(this.name, this.api) - } - - // eslint-disable-next-line require-await - async retrieve(): Promise { - return this.api.get(this.apiPath) - } - - // eslint-disable-next-line require-await - async update( - schema: TypesenseCollectionUpdateSchema - ): Promise { - return this.api.patch(this.apiPath, schema) - } - - // eslint-disable-next-line require-await - async delete(): Promise { - return this.api.delete(this.apiPath) - } - - documents(): TypesenseDocuments { - return this._documents - } -} - -/** - * TypesenseCollections class implements the basic methods of collections - */ -export class TypesenseCollections { - apiPath: string = '/collections' - - constructor(private api: TypesenseApi) {} - - // eslint-disable-next-line require-await - async create(schema: TypesenseCollectionCreateSchema) { - return this.api.post(this.apiPath, schema) - } - - // eslint-disable-next-line require-await - async retrieve() { - return this.api.get(this.apiPath) - } -} - -/** - * Typesense class is used to create a base instance to work with Typesense - * It initiates classes that provides access to methods of collections - * or an individual collection - */ -export class Typesense { - config: TypesenseConfig - api: TypesenseApi - collectionsRecords: Record = {} - private readonly _collections: TypesenseCollections - - constructor(options: TypesenseConfigOptions) { - this.config = new TypesenseConfig(options) - this.api = new TypesenseApi(this.config) - this._collections = new TypesenseCollections(this.api) - } - - collections(): TypesenseCollections - collections(collectionName: string): TypesenseCollection - collections(collectionName?: string): TypesenseCollection | TypesenseCollections { - if (!collectionName) { - return this._collections - } else { - if (this.collectionsRecords[collectionName] === undefined) { - this.collectionsRecords[collectionName] = new TypesenseCollection( - collectionName, - this.api - ) - } - return this.collectionsRecords[collectionName] - } - } -} diff --git a/src/components/database/typesenseApi.ts b/src/components/database/typesenseApi.ts deleted file mode 100644 index 6aba24442..000000000 --- a/src/components/database/typesenseApi.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { setTimeout } from 'timers/promises' -import { TypesenseConfig } from './typesenseConfig.js' -import { TypesenseError } from './typesense.js' -import { TypesenseNode } from '../../@types/index.js' - -/** - * TypesenseApi class is used to implement an api interface - * for working with Typesense via http requests - */ -export class TypesenseApi { - currentNodeIndex = -1 - - constructor(private config: TypesenseConfig) {} - - // eslint-disable-next-line require-await - async get(endpoint: string, queryParameters: any = {}): Promise { - return this.request('get', endpoint, { - queryParameters - }) - } - - // eslint-disable-next-line require-await - async post( - endpoint: string, - bodyParameters: any = {}, - queryParameters: any = {} - ): Promise { - return this.request('post', endpoint, { - queryParameters, - bodyParameters - }) - } - - // eslint-disable-next-line require-await - async delete(endpoint: string, queryParameters: any = {}): Promise { - return this.request('delete', endpoint, { queryParameters }) - } - - // eslint-disable-next-line require-await - async put( - endpoint: string, - bodyParameters: any = {}, - queryParameters: any = {} - ): Promise { - return this.request('put', endpoint, { - queryParameters, - bodyParameters - }) - } - - // eslint-disable-next-line require-await - async patch( - endpoint: string, - bodyParameters: any = {}, - queryParameters: any = {} - ): Promise { - return this.request('patch', endpoint, { - queryParameters, - bodyParameters - }) - } - - getNextNode(): TypesenseNode { - let candidateNode: TypesenseNode = this.config.nodes[0] - if (this.config.nodes.length === 1) { - return candidateNode - } - this.currentNodeIndex = (this.currentNodeIndex + 1) % this.config.nodes.length - candidateNode = this.config.nodes[this.currentNodeIndex] - this.config.logger.debug( - `Updated current node to Node ${candidateNode.host}:${candidateNode.port}` - ) - return candidateNode - } - - async request( - requestType: string, - endpoint: string, - { - queryParameters = null, - bodyParameters = null, - skipConnectionTimeout = false - }: { - queryParameters?: any - bodyParameters?: any - skipConnectionTimeout?: boolean - } - ): Promise { - let lastException - for (let numTries = 1; numTries <= this.config.numRetries + 1; numTries++) { - const node = this.getNextNode() - this.config.logger.debug( - `Request ${endpoint}: Attempting ${requestType.toUpperCase()} request Try #${numTries} to Node ${ - node.host - }` - ) - - try { - const url = new URL(`${node.protocol}://${node.host}:${node.port}${endpoint}`) - if (queryParameters !== null) { - for (const [key, value] of Object.entries(queryParameters)) { - if (value !== undefined && value !== null) { - url.searchParams.set(key, String(value)) - } - } - } - - const init: RequestInit = { - method: requestType.toUpperCase(), - headers: { - 'X-TYPESENSE-API-KEY': this.config.apiKey, - ...(bodyParameters !== null && { 'content-type': 'application/json' }) - }, - // axios sent string bodies raw; only stringify non-string bodies so a - // pre-serialized payload (e.g. JSONL) isn't double-encoded - ...(bodyParameters !== null && { - body: - typeof bodyParameters === 'string' - ? bodyParameters - : JSON.stringify(bodyParameters) - }) - } - - if (skipConnectionTimeout !== true) { - init.signal = AbortSignal.timeout(this.config.connectionTimeoutSeconds * 1000) - } - - const response = await fetch(url, init) - this.config.logger.debug( - `Request ${endpoint}: Request to Node ${node.host} was made. Response Code was ${response.status}.` - ) - - // fetch never throws on HTTP status and doesn't auto-parse — replicate - // axios's old transformResponse (JSON only when content-type says so). - const contentType = response.headers.get('content-type') ?? '' - const data: any = contentType.startsWith('application/json') - ? await response.json() - : await response.text() - - if (response.status >= 200 && response.status < 300) { - return data - } else if (response.status < 500) { - return Promise.reject(this.customError(response.status, data)) - } else { - // non-json error bodies (e.g. a proxy 5xx) arrive as a plain string - throw new Error(typeof data === 'string' ? data : data?.message) - } - } catch (error: any) { - lastException = error - // each failed attempt is a debug line: the request is only a failure once every - // retry is used up, and that is logged once at error below - this.config.logger.debug( - `Request ${endpoint}: Request to Node ${node.host} failed due to "${ - error.cause?.code ?? error.code - } ${error.message}"` - ) - if (numTries <= this.config.numRetries) { - this.config.logger.debug( - `Request ${endpoint}: Sleeping for ${this.config.retryIntervalSeconds}s and then retrying request...` - ) - await setTimeout(this.config.retryIntervalSeconds) - } - } - } - this.config.logger.error( - `Request ${endpoint}: ${requestType.toUpperCase()} failed after ${ - this.config.numRetries + 1 - } attempts: "${lastException?.cause?.code ?? lastException?.code} ${ - lastException?.message - }"` - ) - return Promise.reject(lastException) - } - - customError(status: number, data: any): TypesenseError { - // non-json error bodies (e.g. a proxy 4xx) arrive as a plain string - const message = typeof data === 'string' ? data : data?.message - const error = new TypesenseError(message) - error.httpStatus = status - return error - } -} diff --git a/src/components/database/typesenseConfig.ts b/src/components/database/typesenseConfig.ts deleted file mode 100644 index be56436bf..000000000 --- a/src/components/database/typesenseConfig.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { - TypesenseAbstractLogger, - TypesenseConfigOptions, - TypesenseNode -} from '../../@types' -import { DATABASE_LOGGER } from '../../utils/logging/common.js' - -/** - * TypesenseConfig class is used to specify configuration parameters for Typesense - * as well as handling optional cases - */ -export class TypesenseConfig { - apiKey: string - nodes: TypesenseNode[] - numRetries: number - retryIntervalSeconds: number - connectionTimeoutSeconds: number - logLevel: string - logger: TypesenseAbstractLogger - - constructor(options: TypesenseConfigOptions) { - this.apiKey = options.apiKey - this.nodes = options.nodes || [] - this.numRetries = options.numRetries || 3 - this.connectionTimeoutSeconds = options.connectionTimeoutSeconds || 5 - this.retryIntervalSeconds = options.retryIntervalSeconds || 0.1 - this.logLevel = options.logLevel || 'debug' - this.logger = options.logger || DATABASE_LOGGER - } -} diff --git a/src/components/httpRoutes/aquarius.ts b/src/components/httpRoutes/aquarius.ts deleted file mode 100644 index 1bc3f7f8e..000000000 --- a/src/components/httpRoutes/aquarius.ts +++ /dev/null @@ -1,184 +0,0 @@ -import express from 'express' -import { streamToString } from '../../utils/util.js' -import { Readable } from 'stream' -import { PROTOCOL_COMMANDS } from '../../utils/constants.js' -import { LOG_LEVELS_STR } from '../../utils/logging/Logger.js' -import { FindDdoHandler, ValidateDDOHandler } from '../core/handler/ddoHandler.js' -import { QueryDdoStateHandler, QueryHandler } from '../core/handler/queryHandler.js' -import { HTTP_LOGGER } from '../../utils/logging/common.js' -import { QueryCommand } from '../../@types/commands.js' -import { DatabaseFactory } from '../database/DatabaseFactory.js' -import { SearchQuery } from '../../@types/DDO/SearchQuery.js' - -export const aquariusRoutes = express.Router() - -export const AQUARIUS_API_BASE_PATH = '/api/aquarius' - -aquariusRoutes.get( - `${AQUARIUS_API_BASE_PATH}/assets/ddo/:did{/:force}`, - async (req, res) => { - try { - const { did, force } = req.params - if (!did || !did.startsWith('did:op')) { - res.status(400).send('Missing or invalid required parameter: "did"') - return - } - const forceFlag = force === 'true' - const ddo = await new FindDdoHandler(req.oceanNode).findAndFormatDdo(did, forceFlag) - if (ddo) { - res.json(ddo) - } else { - res.status(404).send('DDO not found') - } - } catch (error) { - HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`) - res.status(500).send('Internal Server Error') - } - } -) - -aquariusRoutes.get( - `${AQUARIUS_API_BASE_PATH}/assets/metadata/:did{/:force}`, - async (req, res) => { - try { - const { did, force } = req.params - if (!did || !did.startsWith('did:op')) { - res.status(400).send('Missing or invalid required parameter: "did"') - return - } - const forceFlag = force === 'true' - const ddo = await new FindDdoHandler(req.oceanNode).findAndFormatDdo(did, forceFlag) - if (ddo) { - res.json(ddo) - } else { - res.status(404).send('DDO not found') - } - } catch (error) { - HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`) - res.status(500).send('Internal Server Error') - } - } -) - -aquariusRoutes.post( - `${AQUARIUS_API_BASE_PATH}/assets/metadata/query`, - async (req, res) => { - try { - const searchQuery: SearchQuery = req.body - if (!searchQuery) { - res.status(400).send('Missing required body') - return - } - - const config = req.oceanNode.getConfig() - const queryStrategy = await DatabaseFactory.createMetadataQuery(config.dbConfig) - const transformedQuery = queryStrategy.buildQuery(searchQuery) - - const result = await new QueryHandler(req.oceanNode).handle({ - query: transformedQuery, - command: PROTOCOL_COMMANDS.QUERY, - caller: req.caller - }) - if (result.stream) { - const queryResult = JSON.parse(await streamToString(result.stream as Readable)) - res.json(queryResult) - } else { - res.status(result.status.httpStatus).send(result.status.error) - } - } catch (error) { - HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`) - res.status(500).send('Internal Server Error') - } - } -) - -aquariusRoutes.get(`${AQUARIUS_API_BASE_PATH}/state/ddo`, async (req, res) => { - try { - const config = req.oceanNode.getConfig() - const queryStrategy = await DatabaseFactory.createDdoStateQuery(config.dbConfig) - - const did = req.query.did ? String(req.query.did) : undefined - const nft = req.query.nft ? String(req.query.nft) : undefined - const txId = req.query.txId ? String(req.query.txId) : undefined - - const queryDdoState: QueryCommand = { - query: queryStrategy.buildQuery(did, nft, txId), - command: PROTOCOL_COMMANDS.QUERY, - caller: req.caller - } - - if (!Object.keys(queryDdoState.query).length) { - res - .status(400) - .send( - 'Missing or invalid required parameters, you need to specify one of: "did", "txId", "nft"' - ) - return - } - - const result = await new QueryDdoStateHandler(req.oceanNode).handle(queryDdoState) - - if (result.stream) { - const queryResult = JSON.parse(await streamToString(result.stream as Readable)) - - if ( - queryResult && - typeof queryResult === 'object' && - queryResult.found !== undefined - ) { - if (queryResult.found > 0 && queryResult.hits && queryResult.hits.length > 0) { - res.json(queryResult.hits[0].document || queryResult.hits[0]) - } else { - res.status(404).send('Not found') - } - } else if (Array.isArray(queryResult)) { - if (queryResult.length > 0) { - res.json(queryResult[0]) - } else { - res.status(404).send('Not found') - } - } else { - HTTP_LOGGER.log( - LOG_LEVELS_STR.LEVEL_DEBUG, - `Query result structure (not found): ${JSON.stringify(queryResult)}` - ) - res.status(404).send('Not found') - } - } else { - res.status(result.status.httpStatus).send(result.status.error) - } - } catch (error) { - HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`) - res.status(500).send('Internal Server Error') - } -}) - -aquariusRoutes.post(`${AQUARIUS_API_BASE_PATH}/assets/ddo/validate`, async (req, res) => { - const node = req.oceanNode - try { - if (!req.body) { - res.status(400).send('Missing DDO object') - return - } - - const result = await new ValidateDDOHandler(node).handle({ - ddo: req.body.ddo || null, - publisherAddress: (req.body.publisherAddress as string) || null, - signature: (req.body.signature as string) || null, - nonce: (req.body.nonce as string) || null, - authorization: req.headers?.authorization, - command: PROTOCOL_COMMANDS.VALIDATE_DDO, - caller: req.caller - }) - - if (result.stream) { - const validationResult = JSON.parse(await streamToString(result.stream as Readable)) - res.json(validationResult) - } else { - res.status(result.status.httpStatus).send(result.status.error) - } - } catch (error) { - HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`) - res.status(500).send('Internal Server Error') - } -}) diff --git a/src/components/httpRoutes/ddo.ts b/src/components/httpRoutes/ddo.ts new file mode 100644 index 000000000..d81d48af0 --- /dev/null +++ b/src/components/httpRoutes/ddo.ts @@ -0,0 +1,138 @@ +import express from 'express' +import { streamToString } from '../../utils/util.js' +import { Readable } from 'stream' +import { + DDO_API_BASE_PATH, + PROTOCOL_COMMANDS, + QUERY_API_PATH +} from '../../utils/constants.js' +import { LOG_LEVELS_STR } from '../../utils/logging/Logger.js' +import { FindDdoHandler, ValidateDDOHandler } from '../core/handler/ddoHandler.js' +import { QueryDdoStateHandler, QueryHandler } from '../core/handler/queryHandler.js' +import { HTTP_LOGGER } from '../../utils/logging/common.js' +import { QueryCommand } from '../../@types/commands.js' +import type { DdoStateKey } from '../database/BaseDatabase.js' + +export const ddoRoutes = express.Router() + +// the query of the node: { type, filter, sort, page, size } +ddoRoutes.post(QUERY_API_PATH, async (req, res) => { + try { + if (!req.body) { + res.status(400).send('Missing required body') + return + } + + const result = await new QueryHandler(req.oceanNode).handle({ + query: req.body, + command: PROTOCOL_COMMANDS.QUERY, + caller: req.caller + }) + if (result.stream) { + const queryResult = JSON.parse(await streamToString(result.stream as Readable)) + res.json(queryResult) + } else { + res.status(result.status.httpStatus).send(result.status.error) + } + } catch (error) { + HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`) + res.status(500).send('Internal Server Error') + } +}) + +// before `/:did`, so "state" is not read as a DID +ddoRoutes.get(`${DDO_API_BASE_PATH}/state`, async (req, res) => { + try { + const did = req.query.did ? String(req.query.did) : undefined + const nft = req.query.nft ? String(req.query.nft) : undefined + const txId = req.query.txId ? String(req.query.txId) : undefined + // with several keys, txId counts first, then nft + const key: DdoStateKey | undefined = txId + ? { txId } + : nft + ? { nft } + : did + ? { did } + : undefined + + if (!key) { + res + .status(400) + .send( + 'Missing or invalid required parameters, you need to specify one of: "did", "txId", "nft"' + ) + return + } + + const queryDdoState: QueryCommand = { + query: key, + command: PROTOCOL_COMMANDS.QUERY, + caller: req.caller + } + const result = await new QueryDdoStateHandler(req.oceanNode).handle(queryDdoState) + + if (result.stream) { + const states = JSON.parse(await streamToString(result.stream as Readable)) + if (states.length > 0) { + res.json(states[0]) + } else { + res.status(404).send('Not found') + } + } else { + res.status(result.status.httpStatus).send(result.status.error) + } + } catch (error) { + HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`) + res.status(500).send('Internal Server Error') + } +}) + +ddoRoutes.post(`${DDO_API_BASE_PATH}/validate`, async (req, res) => { + const node = req.oceanNode + try { + if (!req.body) { + res.status(400).send('Missing DDO object') + return + } + + const result = await new ValidateDDOHandler(node).handle({ + ddo: req.body.ddo || null, + publisherAddress: (req.body.publisherAddress as string) || null, + signature: (req.body.signature as string) || null, + nonce: (req.body.nonce as string) || null, + authorization: req.headers?.authorization, + command: PROTOCOL_COMMANDS.VALIDATE_DDO, + caller: req.caller + }) + + if (result.stream) { + const validationResult = JSON.parse(await streamToString(result.stream as Readable)) + res.json(validationResult) + } else { + res.status(result.status.httpStatus).send(result.status.error) + } + } catch (error) { + HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`) + res.status(500).send('Internal Server Error') + } +}) + +ddoRoutes.get(`${DDO_API_BASE_PATH}/:did{/:force}`, async (req, res) => { + try { + const { did, force } = req.params + if (!did || !did.startsWith('did:op')) { + res.status(400).send('Missing or invalid required parameter: "did"') + return + } + const forceFlag = force === 'true' + const ddo = await new FindDdoHandler(req.oceanNode).findAndFormatDdo(did, forceFlag) + if (ddo) { + res.json(ddo) + } else { + res.status(404).send('DDO not found') + } + } catch (error) { + HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`) + res.status(500).send('Internal Server Error') + } +}) diff --git a/src/components/httpRoutes/index.ts b/src/components/httpRoutes/index.ts index a70a88fba..8ca3d1e4e 100644 --- a/src/components/httpRoutes/index.ts +++ b/src/components/httpRoutes/index.ts @@ -4,7 +4,7 @@ import { getProvidersForStringRoute, getProvidersForStringsRoute } from './dids. import { directCommandRoute } from './commands.js' import { logRoutes } from './logs.js' import { providerRoutes } from './provider.js' -import { aquariusRoutes } from './aquarius.js' +import { ddoRoutes } from './ddo.js' import { rootEndpointRoutes } from './rootEndpoint.js' import { fileInfoRoute } from './fileInfo.js' import { computeRoutes } from './compute.js' @@ -46,12 +46,11 @@ httpRoutes.use(fileInfoRoute) // /api/services/initialize // /api/services/nonce httpRoutes.use(providerRoutes) -// /api/aquarius/assets/ddo/:did -// /api/aquarius/assets/metadata/:did -// /api/aquarius/assets/metadata/query -// /api/aquarius/state/ddo -// /api/aquarius/assets/ddo/validate -httpRoutes.use(aquariusRoutes) +// /api/query +// /api/ddo/state +// /api/ddo/validate +// /api/ddo/:did +httpRoutes.use(ddoRoutes) httpRoutes.use(rootEndpointRoutes) // /api/services/computeEnvironments httpRoutes.use(computeRoutes) diff --git a/src/components/persistentStorage/PersistentStorageFactory.ts b/src/components/persistentStorage/PersistentStorageFactory.ts index ddcc68525..e660ffecc 100644 --- a/src/components/persistentStorage/PersistentStorageFactory.ts +++ b/src/components/persistentStorage/PersistentStorageFactory.ts @@ -6,7 +6,7 @@ import type { PersistentStorageObject } from '../../@types/PersistentStorage.js' -import { SqliteClient } from '../database/sqliteClient.js' +import type { BucketRegistry, BucketRow } from '../database/BucketRegistry.js' import { getAddress } from 'ethers' import { OceanNode } from '../../OceanNode.js' import { checkAddressOnAccessList } from '../../utils/accessList.js' @@ -35,14 +35,6 @@ function parseBucketAccessListsJson(accessListJson: string): AccessList[] { } } -export type BucketRow = { - bucketId: string - owner: string - accessListJson: string - createdAt: number - label: string | null -} - export interface PersistentStorageFileInfo { bucketId: string name: string @@ -67,48 +59,12 @@ export type PersistentStorageBucketRecord = { } export abstract class PersistentStorageFactory { - private db: SqliteClient private node: OceanNode - private dbReady = false - private dbReadyPromise: Promise + private buckets: BucketRegistry - constructor(node: OceanNode) { + constructor(node: OceanNode, buckets: BucketRegistry) { this.node = node - // SqliteClient creates the parent directory on construction. - this.db = new SqliteClient('databases/persistentStorage.sqlite') - this.db.exec(` - CREATE TABLE IF NOT EXISTS persistent_storage_buckets ( - bucketId TEXT PRIMARY KEY, - owner TEXT NOT NULL, - accessListJson TEXT NOT NULL, - createdAt INTEGER NOT NULL, - label TEXT - ); - `) - // Migration: add the label column if it doesn't exist. A fresh table already has it, - // so ALTER throws "duplicate column name" — swallow only that; surface any other - // failure instead of starting with a broken schema. Schema setup is synchronous now, - // so the DB is ready by the time the constructor returns. - try { - this.db.exec(`ALTER TABLE persistent_storage_buckets ADD COLUMN label TEXT`) - } catch (alterErr) { - if (!/duplicate column name/i.test(alterErr.message)) { - throw alterErr - } - } - this.dbReady = true - this.dbReadyPromise = Promise.resolve() - } - - public isDbReady(): boolean { - return this.dbReady - } - - private async ensureDbReady(): Promise { - if (this.dbReady) { - return - } - await this.dbReadyPromise + this.buckets = buckets } /** @@ -227,7 +183,7 @@ export abstract class PersistentStorageFactory { } /** - * Lists buckets for a given owner from the SQLite registry (metadata only). + * Lists buckets for a given owner from the bucket registry (metadata only). * `owner` must already be normalized (e.g. checksummed `getAddress`). * Backends that need setup (e.g. localfs init) should override and call `super.listBuckets(owner)`. */ @@ -272,57 +228,30 @@ export abstract class PersistentStorageFactory { return buckets.filter((_, i) => allowed[i]) } - /* - * NOTE: db* methods are intentionally gated on ensureDbReady() to avoid races - * with constructor-time schema creation. - */ - - async dbUpsertBucket( + dbUpsertBucket( bucketId: string, owner: string, accessListJson: string, createdAt: number, label: string | null ): Promise { - // ON CONFLICT does not touch label, so a re-create never clobbers a rename. - const sql = ` - INSERT INTO persistent_storage_buckets (bucketId, owner, accessListJson, createdAt, label) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(bucketId) DO UPDATE SET accessListJson=excluded.accessListJson; - ` - await this.ensureDbReady() - this.db.run(sql, [bucketId, owner, accessListJson, createdAt, label]) - } - - async dbGetBucket(bucketId: string): Promise { - const sql = `SELECT bucketId, owner, accessListJson, createdAt, label FROM persistent_storage_buckets WHERE bucketId = ?` - await this.ensureDbReady() - const row = this.db.get(sql, [bucketId]) - return row ?? null + return this.buckets.upsertBucket(bucketId, owner, accessListJson, createdAt, label) } - async dbListBucketsByOwner(owner: string): Promise { - const sql = `SELECT bucketId, owner, accessListJson, createdAt, label FROM persistent_storage_buckets WHERE owner = ? ORDER BY createdAt ASC` - await this.ensureDbReady() - return this.db.all(sql, [owner]) + dbGetBucket(bucketId: string): Promise { + return this.buckets.getBucket(bucketId) } - async dbDeleteBucket(bucketId: string): Promise { - const sql = `DELETE FROM persistent_storage_buckets WHERE bucketId = ?` - await this.ensureDbReady() - const { changes } = this.db.run(sql, [bucketId]) - return changes === 1 + dbListBucketsByOwner(owner: string): Promise { + return this.buckets.listBucketsByOwner(owner) } - async dbUpdateBucketLabel( + dbUpdateBucketLabel( bucketId: string, owner: string, label: string | null ): Promise { - const sql = `UPDATE persistent_storage_buckets SET label = ? WHERE bucketId = ? AND owner = ?` - await this.ensureDbReady() - const { changes } = this.db.run(sql, [label, bucketId, owner]) - return changes === 1 + return this.buckets.updateBucketLabel(bucketId, owner, label) } isAllowed(consumerAddress: string, accessLists: AccessList[]): Promise { diff --git a/src/components/persistentStorage/PersistentStorageLocalFS.ts b/src/components/persistentStorage/PersistentStorageLocalFS.ts index 92ac36698..19dde48a5 100644 --- a/src/components/persistentStorage/PersistentStorageLocalFS.ts +++ b/src/components/persistentStorage/PersistentStorageLocalFS.ts @@ -19,14 +19,15 @@ import { PersistentStorageFileInfo } from './PersistentStorageFactory.js' import { OceanNode } from '../../OceanNode.js' +import type { BucketRegistry } from '../database/BucketRegistry.js' import { CORE_LOGGER } from '../../utils/logging/common.js' export class PersistentStorageLocalFS extends PersistentStorageFactory { /* eslint-disable security/detect-non-literal-fs-filename -- localfs backend operates on filesystem paths */ private baseFolder: string - constructor(node: OceanNode) { - super(node) + constructor(node: OceanNode, buckets: BucketRegistry) { + super(node, buckets) const options = node.getConfig().persistentStorage .options as PersistentStorageLocalFSOptions diff --git a/src/components/persistentStorage/PersistentStorageS3.ts b/src/components/persistentStorage/PersistentStorageS3.ts index f5c9912b0..4c8f44303 100644 --- a/src/components/persistentStorage/PersistentStorageS3.ts +++ b/src/components/persistentStorage/PersistentStorageS3.ts @@ -12,11 +12,12 @@ import type { PersistentStorageObject } from '../../@types/PersistentStorage.js' import { OceanNode } from '../../OceanNode.js' +import type { BucketRegistry } from '../database/BucketRegistry.js' export class PersistentStorageS3 extends PersistentStorageFactory { private options: PersistentStorageS3Options - constructor(node: OceanNode) { - super(node) + constructor(node: OceanNode, buckets: BucketRegistry) { + super(node, buckets) this.options = node.getConfig().persistentStorage .options as PersistentStorageS3Options } diff --git a/src/components/persistentStorage/createPersistentStorage.ts b/src/components/persistentStorage/createPersistentStorage.ts index c8588cde4..d1a6735a6 100644 --- a/src/components/persistentStorage/createPersistentStorage.ts +++ b/src/components/persistentStorage/createPersistentStorage.ts @@ -1,20 +1,29 @@ import { OceanNode } from '../../OceanNode.js' +import type { BucketRegistry } from '../database/BucketRegistry.js' import type { PersistentStorageFactory } from './PersistentStorageFactory.js' import { PersistentStorageLocalFS } from './PersistentStorageLocalFS.js' import { PersistentStorageS3 } from './PersistentStorageS3.js' -export function createPersistentStorage(node: OceanNode): PersistentStorageFactory { +export function createPersistentStorage( + node: OceanNode, + buckets: BucketRegistry | undefined +): PersistentStorageFactory { const config = node.getConfig().persistentStorage if (!config?.enabled) { throw new Error('Persistent storage is disabled') } + if (!buckets) { + throw new Error( + 'Persistent storage is enabled but the database is not available: the bucket registry is stored in the database' + ) + } switch (config.type) { case 'localfs': - return new PersistentStorageLocalFS(node) + return new PersistentStorageLocalFS(node, buckets) case 's3': - return new PersistentStorageS3(node) + return new PersistentStorageS3(node, buckets) default: throw new Error( `Unsupported persistent storage type: ${(config as { type?: string })?.type}` diff --git a/src/helpers/scripts/setupNodeEnv.sh b/src/helpers/scripts/setupNodeEnv.sh index 7c2447f08..9eb3dde48 100755 --- a/src/helpers/scripts/setupNodeEnv.sh +++ b/src/helpers/scripts/setupNodeEnv.sh @@ -35,18 +35,49 @@ wallet_file_path=$generated_files_path$wallet_file pk_file_path+=$generated_files_path$pk_file +#add NAME=VALUE to the env file, quoted so that 'source .env' keeps the value as typed +set_env_value() { + printf 'export %s=%q\n' "$1" "$2" >> "$env_file_path" +} #configure database configure_database() { echo "------------------------ Configure Database ----------------------------------" - read -p "Database URL? [ http://localhost:8108/?apiKey=xyz ]: " DB_URL - DB_URL=${DB_URL:-http://localhost:8108/?apiKey=xyz} - REPLACE_STR="DB_URL=$DB_URL" - if [ "$(uname)" == "Darwin" ]; then - sed -i '' -e 's;DB_URL=;'$REPLACE_STR';' "$env_file_path" + while true; do + IFS= read -r -p "Database type? sqlite, postgresql or elasticsearch [ sqlite ]: " DB_TYPE + DB_TYPE=${DB_TYPE:-sqlite} + case "$DB_TYPE" in + sqlite|postgresql|elasticsearch) break ;; + *) echo "Unknown database type: $DB_TYPE" ;; + esac + done + #always set: the node does not start without DB_TYPE + set_env_value DB_TYPE "$DB_TYPE" + if [ "$DB_TYPE" == "sqlite" ]; then + #the default: the node keeps its files in ./databases and needs no connection values + echo "SQLite needs no more settings." + echo "------------------------------------------------------------------------------" + return + fi + if [ "$DB_TYPE" == "postgresql" ]; then + DEFAULT_DB_URL=postgresql://localhost:5432/ocean + DEFAULT_DB_USERNAME=ocean else - sed -i -e 's;DB_URL=;'$REPLACE_STR';' "$env_file_path" + DEFAULT_DB_URL=http://localhost:9200 + DEFAULT_DB_USERNAME=elastic fi - echo "DB_URL final configuration: $REPLACE_STR" + IFS= read -r -p "Database URL? [ $DEFAULT_DB_URL ]: " DB_URL + DB_URL=${DB_URL:-$DEFAULT_DB_URL} + #not sed: a '&' or ';' in the URL (query parameters) would change the replacement + set_env_value DB_URL "$DB_URL" + echo "DB_URL final configuration: DB_URL=$DB_URL" + #the node does not start on a database server without its credentials + IFS= read -r -p "Database username? [ $DEFAULT_DB_USERNAME ]: " DB_USERNAME + DB_USERNAME=${DB_USERNAME:-$DEFAULT_DB_USERNAME} + set_env_value DB_USERNAME "$DB_USERNAME" + IFS= read -r -p "Database password? [ changeme ]: " DB_PASSWORD + DB_PASSWORD=${DB_PASSWORD:-changeme} + set_env_value DB_PASSWORD "$DB_PASSWORD" + echo "DB_USERNAME final configuration: DB_USERNAME=$DB_USERNAME" echo "------------------------------------------------------------------------------" } #configure a basic RPC setting @@ -227,11 +258,8 @@ else exit 1 fi -read -p "Do you want to run a database on your node? [ y/n ]: " run_database -run_database=${run_database:-n} -if [ "$run_database" == 'y' ]; then - configure_database -fi +#the node does not start without a database +configure_database read -p "Do you want to index a network on your node? [ y/n ]: " run_indexer run_indexer=${run_indexer:-n} diff --git a/src/index.ts b/src/index.ts index f2ef4759e..3929aa509 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,11 +7,7 @@ import { OceanNode } from './OceanNode.js' import { KeyManager } from './components/KeyManager/index.js' import { BlockchainRegistry } from './components/BlockchainRegistry/index.js' import { httpRoutes } from './components/httpRoutes/index.js' -import { - getConfiguration, - computeCodebaseHash, - ENVIRONMENT_VARIABLES -} from './utils/index.js' +import { getConfiguration, computeCodebaseHash } from './utils/index.js' import { enableFileLoggingFallback, @@ -31,7 +27,6 @@ import { fileURLToPath } from 'url' import cors from 'cors' import { scheduleCronJobs } from './utils/cronjobs/scheduleCronJobs.js' import { requestValidator } from './components/httpRoutes/requestValidator.js' -import { hasValidDBConfiguration, isReachableConnection } from './utils/database.js' import type { OceanNodeDBConfig } from './@types/OceanNode.js' // Telemetry gauge registration. These modules depend on `@opentelemetry/api` only (no SDK), so // importing them here does not pull the OTel SDK into the graph ahead of the `--import` bootstrap @@ -41,7 +36,7 @@ import { registerComputeGauges } from './telemetry/computeGauges.js' const app: Express = express() -// Database services (Elasticsearch/Typesense) frequently take longer to accept +// A database server frequently takes longer to accept // connections than the node itself takes to boot. Database.init() gives // up after a single failed attempt, which leaves the node running permanently // without Indexer and without C2D. @@ -53,33 +48,19 @@ async function initDatabaseWithRetry( maxRetryDelay: number ): Promise { for (let attempt = 1; attempt <= maxAttempts; attempt++) { - const isLastAttempt = attempt === maxAttempts - const notReachable = - !isLastAttempt && - hasValidDBConfiguration(dbConfig) && - !(await isReachableConnection(dbConfig.url)) - - if (!notReachable) { - // On the final attempt allow a partial (DEGRADED) init: if the metadata DB is still - // unreachable we keep the working SQLite core (C2D / nonce / auth) instead of returning - // null. Earlier attempts keep returning null on metadata failure so we retry with backoff - // and give a slow-starting metadata DB time to come up (yielding a full node). - const database = await Database.init(dbConfig, isLastAttempt) - if (database) { - if (attempt > 1) { - OCEAN_NODE_LOGGER.info(`Database initialized after ${attempt} attempts`) - } - return database + const database = await Database.init(dbConfig) + if (database) { + if (attempt > 1) { + OCEAN_NODE_LOGGER.info(`Database initialized after ${attempt} attempts`) } + return database } - if (isLastAttempt) { + if (attempt === maxAttempts) { break } const delay = Math.min(retryDelay * 2 ** (attempt - 1), maxRetryDelay) OCEAN_NODE_LOGGER.warn( - `Database ${ - notReachable ? 'not reachable yet' : 'initialization failed' - } (attempt ${attempt}/${maxAttempts}), retrying in ${delay}ms` + `Database initialization failed (attempt ${attempt}/${maxAttempts}), retrying in ${delay}ms` ) await new Promise((resolve) => setTimeout(resolve, delay)) } @@ -176,8 +157,6 @@ if (!config) { } let node: OceanP2P = null let indexer = null -let provider = null -// If there is no DB URL only the nonce database will be available const dbconn: Database | null = await initDatabaseWithRetry( config.dbConfig, config.dbInitMaxAttempts, @@ -185,36 +164,16 @@ const dbconn: Database | null = await initDatabaseWithRetry( config.dbInitMaxRetryDelay ) if (!dbconn) { - OCEAN_NODE_LOGGER.error('Database failed to initialize') -} else if (hasValidDBConfiguration(config.dbConfig) && !dbconn.metadataInitialized) { - // DEGRADED mode: SQLite core (C2D / nonce / auth) is up but the metadata DB (Typesense/ - // Elasticsearch) is unreachable. The Indexer eagerly touches the metadata databases in its - // constructor, so it must not be started here. - config.hasIndexer = false - OCEAN_NODE_LOGGER.warn( - 'Running in DEGRADED mode: metadata database is unavailable, Indexer is disabled. C2D / nonce / auth remain available.' - ) -} - -// LOG_DB is on but there is no usable log database (no/invalid DB config, failed init or -// DEGRADED mode): keep the logs on disk instead -if (USE_DB_TRANSPORT() && !dbconn?.logs) { - const reason = dbconn ? 'log DB not initialized' : 'DB init failed' - // LOG_FILES=false as well: never leave the node without any log output - if (!enableFileLoggingFallback(reason)) ensureLogOutput(reason) -} - -if (!hasValidDBConfiguration(config.dbConfig)) { - config.hasIndexer = false - if (config.dbConfig?.url) { - OCEAN_NODE_LOGGER.warn( - `Invalid metadata DB configuration (check ${ENVIRONMENT_VARIABLES.DB_URL.name}, ${ENVIRONMENT_VARIABLES.DB_TYPE.name} and credentials): Indexer disabled` - ) - } else { - OCEAN_NODE_LOGGER.info( - `${ENVIRONMENT_VARIABLES.DB_URL.name} not set: running without metadata DB (Indexer disabled)` - ) + // LOG_DB is on but there is no log database: write the failure to disk, or to the + // console when LOG_FILES=false, so it is never lost + if (USE_DB_TRANSPORT()) { + const reason = 'DB init failed' + if (!enableFileLoggingFallback(reason)) ensureLogOutput(reason) } + // all state is in the database: exit so the orchestrator restarts the node + OCEAN_NODE_LOGGER.error('Database failed to initialize, exiting') + await flushLogs() + process.exit(1) } OCEAN_NODE_LOGGER.info(getLoggingSummary()) @@ -224,22 +183,16 @@ const keyManager = new KeyManager(config) const blockchainRegistry = new BlockchainRegistry(keyManager, config) if (config.hasP2P) { - if (dbconn) { - node = new OceanP2P(config, keyManager, dbconn) - } else { - node = new OceanP2P(config, keyManager) - } + node = new OceanP2P(config, keyManager, dbconn) await node.start() // Attach the P2P observable-gauge callbacks now that libp2p is up. No-ops cleanly if P2P is // disabled (this block only runs when hasP2P) or when telemetry is unconfigured. registerP2PGauges(node) } -if (config.hasIndexer && dbconn) { +if (config.hasIndexer) { indexer = new OceanIndexer(dbconn, config, blockchainRegistry) } -if (dbconn) { - provider = new OceanProvider(dbconn) -} +const provider = new OceanProvider(dbconn) // Singleton instance across application const oceanNode = OceanNode.getInstance( diff --git a/src/test/.env.test b/src/test/.env.test index 0d95e5057..fae3e7eda 100644 --- a/src/test/.env.test +++ b/src/test/.env.test @@ -3,9 +3,6 @@ P2P_ipV4BindTcpPort=8000 PRIVATE_KEY=0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58 RPCS='{ "8996": {"rpc": "http://127.0.0.1:8545", "chainId": 8996, "network": "development", "chunkSize": 100}}' INDEXER_NETWORKS='[8996]' -DB_URL=http://localhost:9200 -DB_USERNAME=elastic -DB_PASSWORD=changeme IPFS_GATEWAY=https://ipfs.io/ ARWEAVE_GATEWAY=https://arweave.net/ NODE1_PRIVATE_KEY=0xcb345bd2b11264d523ddaf383094e2675c420a17511c3102a53817f13474a7ff @@ -13,4 +10,3 @@ NODE2_PRIVATE_KEY=0x3634cc4a3d2694a1186a7ce545f149e022eea103cc254d18d08675104bb4 INDEXER_INTERVAL=9000 #ADDRESS_FILE=${HOME}/.ocean/ocean-contracts/artifacts/address.json LOG_LEVEL=debug -DB_TYPE=elasticsearch diff --git a/src/test/config.json b/src/test/config.json index 0cc341719..0bededa6d 100644 --- a/src/test/config.json +++ b/src/test/config.json @@ -59,12 +59,6 @@ "enableNetworkStats": false }, "httpPort": 8001, - "dbConfig": { - "url": "http://localhost:9200", - "username": "", - "password": "", - "dbType": "elasticsearch" - }, "supportedNetworks": { "8996": { "rpc": "http://127.0.0.1:8545", diff --git a/src/test/data/ddoSchema.ts b/src/test/data/ddoSchema.ts deleted file mode 100644 index fa024a22a..000000000 --- a/src/test/data/ddoSchema.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { TypesenseCollectionCreateSchema } from '../../@types' - -export const ddoSchema: TypesenseCollectionCreateSchema = { - name: 'ddo', - enable_nested_fields: true, - fields: [ - { name: '@context', type: 'string[]', optional: true }, - { name: 'chainId', type: 'int64', optional: true }, - { name: 'version', type: 'string', sort: true, optional: true }, - { name: 'nftAddress', type: 'string' }, - - { name: 'metadata.description', type: 'string' }, - { name: 'metadata.copyrightHolder', type: 'string', optional: true }, - { name: 'metadata.name', type: 'string', optional: true }, - { name: 'metadata.type', type: 'string', optional: true }, - { name: 'metadata.author', type: 'string', optional: true }, - { name: 'metadata.license', type: 'string', optional: true }, - { name: 'metadata.links', type: 'string', optional: true }, - { name: 'metadata.tags', type: 'string[]', optional: true }, - { name: 'metadata.categories', type: 'string', optional: true }, - { name: 'metadata.contentLanguage', type: 'string', optional: true }, - { name: 'metadata.algorithm.version', type: 'string', optional: true }, - { name: 'metadata.algorithm.language', type: 'string', optional: true }, - { name: 'metadata.algorithm.container.entrypoint', type: 'string', optional: true }, - { name: 'metadata.algorithm.container.image', type: 'string', optional: true }, - { name: 'metadata.algorithm.container.tag', type: 'string', optional: true }, - { name: 'metadata.algorithm.container.checksum', type: 'string', optional: true } - ] -} diff --git a/src/test/data/nonceSchema.ts b/src/test/data/nonceSchema.ts deleted file mode 100644 index f855b8024..000000000 --- a/src/test/data/nonceSchema.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { TypesenseCollectionCreateSchema } from '../../@types' - -export const nonceSchema: TypesenseCollectionCreateSchema = { - name: 'nonce', - enable_nested_fields: true, - fields: [ - { name: 'id', type: 'string' }, - { name: 'nonce', type: 'int64', sort: true } // store nonce as string - ] -} diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index 67ea01733..e191544d3 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -26,7 +26,6 @@ import { type DBComputeJob } from '../../@types/C2D/C2D.js' import { - // DB_TYPES, ENVIRONMENT_VARIABLES, EVENTS, PROTOCOL_COMMANDS, diff --git a/src/test/integration/configDatabase.test.ts b/src/test/integration/configDatabase.test.ts index e1d56e6a4..479324a3f 100644 --- a/src/test/integration/configDatabase.test.ts +++ b/src/test/integration/configDatabase.test.ts @@ -1,156 +1,36 @@ import { Database } from '../../components/database/index.js' -import { OceanNode } from '../../OceanNode.js' import { expect, assert } from 'chai' -import { OceanIndexer } from '../../components/Indexer/index.js' -import { - buildEnvOverrideConfig, - getMockSupportedNetworks, - OverrideEnvConfig, - setupEnvironment, - tearDownEnvironment, - TEST_ENV_CONFIG_FILE -} from '../utils/utils.js' -import { ENVIRONMENT_VARIABLES, getConfiguration } from '../../utils/index.js' -import { SQLLiteConfigDatabase } from '../../components/database/SQLLiteConfigDatabase.js' -import { DB_TYPES } from '../../utils/constants.js' -import { OceanNodeDBConfig } from '../../@types/OceanNode.js' -import { homedir } from 'os' - -const versionConfig: OceanNodeDBConfig = { - url: 'http://localhost:8108/test-version?apiKey=xyz', - dbType: DB_TYPES.TYPESENSE -} - -const emptyDBConfig: OceanNodeDBConfig = { - url: '', - dbType: null -} +import { getConfiguration } from '../../utils/index.js' +// The config store only: no indexer runs here. A stop during the start of an indexer does +// not cancel the start, so an indexer started here would keep running into the next suite. describe('********** Config Database', () => { let database: Database - let oceanIndexer: OceanIndexer - let initialVersionNull: any - let previousConfiguration: OverrideEnvConfig[] - let oceanNode: OceanNode before(async () => { - database = await Database.init(versionConfig) - // override and save configuration (always before calling getConfig()) - previousConfiguration = await setupEnvironment( - TEST_ENV_CONFIG_FILE, - buildEnvOverrideConfig( - [ - ENVIRONMENT_VARIABLES.RPCS, - ENVIRONMENT_VARIABLES.INDEXER_NETWORKS, - ENVIRONMENT_VARIABLES.PRIVATE_KEY, - ENVIRONMENT_VARIABLES.AUTHORIZED_DECRYPTERS, - ENVIRONMENT_VARIABLES.ALLOWED_ADMINS, - ENVIRONMENT_VARIABLES.ADDRESS_FILE - ], - [ - JSON.stringify(getMockSupportedNetworks()), - JSON.stringify([8996]), - '0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58', - JSON.stringify(['0xe2DD09d719Da89e5a3D0F2549c7E24566e947260']), - JSON.stringify(['0xe2DD09d719Da89e5a3D0F2549c7E24566e947260']), - `${homedir}/.ocean/ocean-contracts/artifacts/address.json` - ] - ) - ) - - it('should have null version initially', async () => { - initialVersionNull = await oceanIndexer.getDatabase().sqliteConfig.retrieveValue() - assert(initialVersionNull.value === null, 'Initial version should be null') - }) - const config = await getConfiguration(true) - oceanNode = await OceanNode.getInstance( - config, - database, - null, - null, - null, - null, - null, - true - ) - oceanIndexer = new OceanIndexer(database, config, oceanNode.blockchainRegistry) - oceanNode.addIndexer(oceanIndexer) - }) - after(async () => { - await oceanNode.tearDownAll() - await tearDownEnvironment(previousConfiguration) - }) - - it('check version DB instance of SQL Lite', () => { - expect(database.sqliteConfig).to.be.instanceOf(SQLLiteConfigDatabase) + database = await Database.init((await getConfiguration()).dbConfig) }) - // Use a dedicated config key (not 'version'): the live OceanIndexer started by this suite - // writes the running node version to the 'version' key in the shared config.sqlite during its - // background reindex (see checkAndTriggerReindexing), which would race these CRUD assertions. const TEST_CONFIG_KEY = 'test-config-version' - it('should set and retrieve version', async () => { - // Set a specific test version - const testVersion = '0.9.9' - await oceanIndexer - .getDatabase() - .sqliteConfig.createOrUpdateConfig(TEST_CONFIG_KEY, testVersion) - - // Verify we can retrieve it - const version = await oceanIndexer - .getDatabase() - .sqliteConfig.retrieveValue(TEST_CONFIG_KEY) - assert(version.value === testVersion, `Version should be ${testVersion}`) + it('should return { value: null } for an absent key', async () => { + // a new key on each run: a database server keeps the keys of the earlier runs + const version = await database.config.retrieveValue(`test-absent-${Date.now()}`) + assert(version !== null, 'An absent key is not a DB error') + assert(version.value === null, 'An absent key should have a null value') }) - it('should update version and retrieve latest', async () => { - const initialVersion = '0.2.2' - const updatedVersion = '0.2.3' - - // Set initial version - await oceanIndexer - .getDatabase() - .sqliteConfig.createOrUpdateConfig(TEST_CONFIG_KEY, initialVersion) - let version = await oceanIndexer - .getDatabase() - .sqliteConfig.retrieveValue(TEST_CONFIG_KEY) - assert(version.value === initialVersion, `Version should be ${initialVersion}`) - - // Update to new version - await oceanIndexer - .getDatabase() - .sqliteConfig.createOrUpdateConfig(TEST_CONFIG_KEY, updatedVersion) - version = await oceanIndexer.getDatabase().sqliteConfig.retrieveValue(TEST_CONFIG_KEY) - assert(version.value === updatedVersion, `Version should be ${updatedVersion}`) - }) -}) - -describe('VersionDatabase CRUD (without Elastic or Typesense config)', () => { - let database: Database - - before(async () => { - database = await Database.init(emptyDBConfig) - }) - - it('check version DB instance of SQL Lite', () => { - expect(database.sqliteConfig).to.be.instanceOf(SQLLiteConfigDatabase) - }) - - // Dedicated key (not 'version'): isolates these CRUD assertions from the node 'version' key - // that a background indexer reindex may write to the shared config.sqlite. - const TEST_CONFIG_KEY = 'test-crud-version' - - it('create version', async () => { - const result = await database.sqliteConfig.createOrUpdateConfig( - TEST_CONFIG_KEY, - '0.1.0' - ) + it('create and retrieve version', async () => { + const created = await database.config.createOrUpdateConfig(TEST_CONFIG_KEY, '0.1.0') + expect(created).to.deep.equal({ key: TEST_CONFIG_KEY, value: '0.1.0' }) + const result = await database.config.retrieveValue(TEST_CONFIG_KEY) expect(result?.value).to.equal('0.1.0') }) - it('retrieve latest version', async () => { - const result = await database.sqliteConfig.retrieveValue(TEST_CONFIG_KEY) - expect(result?.value).to.equal('0.1.0') + it('should update version and retrieve latest', async () => { + await database.config.createOrUpdateConfig(TEST_CONFIG_KEY, '0.2.2') + expect((await database.config.retrieveValue(TEST_CONFIG_KEY)).value).to.equal('0.2.2') + await database.config.createOrUpdateConfig(TEST_CONFIG_KEY, '0.2.3') + expect((await database.config.retrieveValue(TEST_CONFIG_KEY)).value).to.equal('0.2.3') }) }) diff --git a/src/test/integration/credentials.test.ts b/src/test/integration/credentials.test.ts index b2e2fed48..73fd0cf63 100644 --- a/src/test/integration/credentials.test.ts +++ b/src/test/integration/credentials.test.ts @@ -20,7 +20,7 @@ import { OceanIndexer } from '../../components/Indexer/index.js' import { OceanNode } from '../../OceanNode.js' import { RPCS, SupportedNetwork } from '../../@types/blockchain.js' import { streamToObject } from '../../utils/util.js' -import { expectedTimeoutFailure, waitToIndex } from './testUtils.js' +import { expectedTimeoutFailure, waitToIndex, waitForCondition } from './testUtils.js' import { Blockchain, @@ -69,6 +69,7 @@ import { createHashForSignature, safeSign } from '../utils/signature.js' describe('********** [Credentials Flow] - Should run a complete node flow.', () => { let config: OceanNodeConfig let oceanNode: OceanNode + let database: Database let provider: JsonRpcProvider let computeEnvironments: any let firstEnv: any @@ -135,7 +136,18 @@ describe('********** [Credentials Flow] - Should run a complete node flo ) config = await getConfiguration(true) // Force reload the configuration - const database = await Database.init(config.dbConfig) + database = await Database.init(config.dbConfig) + + // Stop any indexer left running by a previous integration test file before we spin up our + // own. Integration suites share one process, one DB and the OceanNode singleton, so a stale + // indexer constructed WITHOUT this suite's AUTHORIZED_PUBLISHERS config keeps polling chain + // 8996 and would index assets this suite expects to be rejected — the root cause of the flaky + // "should NOT allow to index ... AUTHORIZED_PUBLISHERS" failures. + const oldIndexer = OceanNode.getInstance(config, database).getIndexer() + if (oldIndexer) { + await oldIndexer.stopAllChainIndexers() + } + oceanNode = OceanNode.getInstance( config, database, @@ -146,6 +158,9 @@ describe('********** [Credentials Flow] - Should run a complete node flo null, true ) + // Wait for the previous instance's async teardown (scheduled by getInstance above) to finish + // so the old and new indexers never overlap while crawling. + await OceanNode.awaitPendingTeardown() const indexer = new OceanIndexer(database, config, oceanNode.blockchainRegistry) oceanNode.addIndexer(indexer) await oceanNode.addC2DEngines() @@ -598,25 +613,47 @@ describe('********** [Credentials Flow] - Should run a complete node flo const nonAuthorizedAccount = (await provider.getSigner(4)) as Signer const authorizedAccount = await publisherAccount.getAddress() - expect( - config.authorizedPublishers.length === 1 && - config.authorizedPublishers[0] === authorizedAccount, - 'Unable to set AUTHORIZED_PUBLISHERS' + // #3: the previous `expect(boolean, msg)` had no matcher, so it asserted nothing — a + // mis-applied AUTHORIZED_PUBLISHERS override passed silently and left the rest of the test + // meaningless. Assert the config for real. + expect(config.authorizedPublishers.length).to.equal(1) + expect(config.authorizedPublishers[0].toLowerCase()).to.equal( + authorizedAccount.toLowerCase() ) const publishedDataset = await publishAsset( downloadAssetWithCredentials, nonAuthorizedAccount ) + const rejectedDid = publishedDataset?.ddo.id - // will timeout - const { ddo, wasTimeout } = await waitToIndex( + // It must NOT land in the DDO collection (this wait times out on success). + const { ddo: indexedDdo } = await waitToIndex( oceanNode, - publishedDataset?.ddo.id, + rejectedDid, EVENTS.METADATA_CREATED, DEFAULT_TEST_TIMEOUT ) - - assert(ddo === null && wasTimeout === true, 'DDO should NOT have been indexed') + assert(indexedDdo === null, 'DDO should NOT have been indexed') + + // #2: assert the gate POSITIVELY rejected this publisher instead of only asserting absence + // from the collection — "absent" also holds when the event was simply never processed, which + // is exactly what masked the flake. The MetadataEventProcessor records an invalid ddoState + // ("... is NOT part of the AUTHORIZED_PUBLISHERS group.") when it rejects, so require that + // explicit record: a deterministic signal that this node's gate actually fired. + const rejectedState = await waitForCondition(async () => { + const state = await database.ddoState.retrieve(rejectedDid) + return state && state.valid === false ? state : null + }, DEFAULT_TEST_TIMEOUT) + assert( + rejectedState && rejectedState.valid === false, + 'DDO should be recorded as rejected: publisher not on AUTHORIZED_PUBLISHERS' + ) + assert( + String(rejectedState.error).includes( + ENVIRONMENT_VARIABLES.AUTHORIZED_PUBLISHERS.name + ), + 'ddoState rejection reason should reference AUTHORIZED_PUBLISHERS' + ) }) }) diff --git a/src/test/integration/database.test.ts b/src/test/integration/database.test.ts index 2de93c3bb..e828ea6e3 100644 --- a/src/test/integration/database.test.ts +++ b/src/test/integration/database.test.ts @@ -1,43 +1,22 @@ -import { SearchQuery } from '../../@types/DDO/SearchQuery.js' -import { AbstractOrderDatabase } from '../../components/database/BaseDatabase.js' -import { DatabaseFactory } from '../../components/database/DatabaseFactory.js' +import { describeOnElasticsearch } from '../utils/utils.js' import { Database } from '../../components/database/index.js' +import type { Client } from '@elastic/elasticsearch' +import { Wallet } from 'ethers' import { expect, assert } from 'chai' -import { DB_TYPES } from '../../utils/constants.js' import { OceanNodeDBConfig } from '../../@types/OceanNode.js' import { isDefined } from '../../utils/util.js' -import { SQLLiteNonceDatabase } from '../../components/database/SQLLiteNonceDatabase.js' - -const typesenseConfig: OceanNodeDBConfig = { - url: 'http://localhost:8108/?apiKey=xyz', - dbType: DB_TYPES.TYPESENSE -} +import { ElasticsearchNonceDatabase } from '../../components/database/ElasticSearchDatabase.js' +import { createElasticsearchClientWithRetry } from '../../components/database/ElasticsearchConfigHelper.js' +import { elasticSchemas } from '../../components/database/ElasticSchemas.js' const elasticConfig: OceanNodeDBConfig = { + dbType: 'elasticsearch', url: 'http://localhost:9200', - dbType: DB_TYPES.ELASTIC_SEARCH, username: process.env.DB_USERNAME, password: process.env.DB_PASSWORD } -const emptyDBConfig: OceanNodeDBConfig = { - url: '', - dbType: null -} - -describe('********** Database', () => { - let database: Database - - before(async () => { - database = await Database.init(typesenseConfig) - }) - - it('instance Database', () => { - expect(database).to.be.instanceOf(Database) - }) -}) - -describe('********** DdoDatabase CRUD', () => { +describeOnElasticsearch('********** DdoDatabase CRUD', () => { let database: Database const ddoWithInvalidDid = { hashType: 'sha256', @@ -64,7 +43,7 @@ describe('********** DdoDatabase CRUD', () => { } before(async () => { - database = await Database.init(typesenseConfig) + database = await Database.init(elasticConfig) }) it('creates ddo schema as an array', () => { @@ -73,11 +52,7 @@ describe('********** DdoDatabase CRUD', () => { assert(Array.isArray(ddoSchemas)) assert(ddoSchemas.length > 1) for (const ddoSchema of ddoSchemas) { - if (database.ddo.isTypesenseSchema(ddoSchema)) { - assert(ddoSchema.name) - assert(ddoSchema.fields) - assert(ddoSchema.fields.length > 0) - } + assert(ddoSchema.index) } }) @@ -87,15 +62,27 @@ describe('********** DdoDatabase CRUD', () => { }) }) -describe('********** NonceDatabase CRUD - SQL lite (With typesense DB config)', () => { +describeOnElasticsearch('********** NonceDatabase CRUD', () => { let database: Database before(async () => { - database = await Database.init(typesenseConfig) + database = await Database.init(elasticConfig) + }) + + beforeEach(async () => { + const client = await createElasticsearchClientWithRetry(elasticConfig) + // nonce writes do not refresh; make them visible to the deleteByQuery search + await client.indices.refresh({ index: elasticSchemas.nonceSchemas.index }) + await client.deleteByQuery({ + index: elasticSchemas.nonceSchemas.index, + query: { match_all: {} }, + refresh: true, + conflicts: 'proceed' + }) }) - it('check nonce DB instance of SQL Lite', () => { - expect(database.nonce).to.be.instanceOf(SQLLiteNonceDatabase) + it('nonce DB is the Elasticsearch store', () => { + expect(database.nonce).to.be.instanceOf(ElasticsearchNonceDatabase) }) it('create nonce', async () => { @@ -105,57 +92,34 @@ describe('********** NonceDatabase CRUD - SQL lite (With typesense DB co }) it('retrieve nonce', async () => { + expect(await database.nonce.retrieve('0x123')).to.deep.equal({ + id: '0x123', + nonce: null + }) + await database.nonce.create('0x123', 0) const result = await database.nonce.retrieve('0x123') expect(result?.id).to.equal('0x123') expect(result?.nonce).to.equal(0) }) it('update nonce', async () => { - const result = await database.nonce.update('0x123', 1) - expect(result?.id).to.equal('0x123') - expect(result?.nonce).to.equal(1) - }) - - it('delete nonce', async () => { - const result = await database.nonce.delete('0x123') - expect(result?.id).to.equal('0x123') - expect(result?.nonce).to.equal(1) - }) -}) - -describe('********** NonceDatabase CRUD (without Elastic or Typesense config)', () => { - let database: Database - - before(async () => { - database = await Database.init(emptyDBConfig) - }) - - it('check nonce DB instance of SQL Lite', () => { - expect(database.nonce).to.be.instanceOf(SQLLiteNonceDatabase) - }) - - it('create nonce', async () => { - const result = await database.nonce.create('0x456', 0) - expect(result?.id).to.equal('0x456') - expect(result?.nonce).to.equal(0) - }) - - it('retrieve nonce', async () => { - const result = await database.nonce.retrieve('0x456') - expect(result?.id).to.equal('0x456') - expect(result?.nonce).to.equal(0) - }) - - it('update nonce', async () => { + await database.nonce.create('0x456', 0) const result = await database.nonce.update('0x456', 1) expect(result?.id).to.equal('0x456') expect(result?.nonce).to.equal(1) + expect((await database.nonce.retrieve('0x456'))?.nonce).to.equal(1) }) it('delete nonce', async () => { + await database.nonce.create('0x456', 1) const result = await database.nonce.delete('0x456') expect(result?.id).to.equal('0x456') expect(result?.nonce).to.equal(1) + expect((await database.nonce.retrieve('0x456'))?.nonce).to.equal(null) + expect(await database.nonce.delete('0x456')).to.deep.equal({ + id: '0x456', + nonce: null + }) }) it('keys rows by the checksummed address and reads them back in any casing', async () => { @@ -172,12 +136,12 @@ describe('********** NonceDatabase CRUD (without Elastic or Typesense co }) }) -describe('********** IndexerDatabase CRUD', () => { +describeOnElasticsearch('********** IndexerDatabase CRUD', () => { let database: Database let existsPrevious: any = {} before(async () => { - database = await Database.init(typesenseConfig) + database = await Database.init(elasticConfig) }) it('create indexer', async () => { @@ -189,14 +153,14 @@ describe('********** IndexerDatabase CRUD', () => { expect(result?.lastIndexedBlock).to.equal(0) } else { existsPrevious = exists - expect(existsPrevious?.id).to.equal('1') + expect(existsPrevious?.lastIndexedBlock).to.be.a('number') } }) + // retrieve returns the stored document, which has no id field it('retrieve indexer', async () => { const result = await database.indexer.retrieve(1) - expect(result?.id).to.equal('1') - if (existsPrevious?.id) { + if (existsPrevious?.lastIndexedBlock !== undefined) { expect(result?.lastIndexedBlock).to.equal(existsPrevious.lastIndexedBlock) } else { expect(result?.lastIndexedBlock).to.equal(0) @@ -211,16 +175,16 @@ describe('********** IndexerDatabase CRUD', () => { it('delete indexer', async () => { const result = await database.indexer.delete(1) - expect(result?.id).to.equal('1') - expect(result?.lastIndexedBlock).to.equal(1) + expect(result).to.deep.equal({ id: '1' }) + expect(await database.indexer.retrieve(1)).to.equal(null) }) }) -describe('********** OrderDatabase CRUD', () => { +describeOnElasticsearch('********** OrderDatabase CRUD', () => { let database: Database before(async () => { - database = await Database.init(typesenseConfig) + database = await Database.init(elasticConfig) }) it('create order', async () => { @@ -234,7 +198,7 @@ describe('********** OrderDatabase CRUD', () => { '0x1', 'did' ) - expect(result?.id).to.equal('order1.0') + expect(result?.orderId).to.equal('order1.0') expect(result?.consumer).to.equal('0x1234') expect(result?.payer).to.equal('0x4567') expect(result?.type).to.equal('startOrder') @@ -259,7 +223,7 @@ describe('********** OrderDatabase CRUD', () => { '0x1235', '0x4567' ) - expect(result?.id).to.equal('order1.0') + // update returns the changed fields only expect(result?.consumer).to.equal('0x1235') expect(result?.payer).to.equal('0x4567') expect(result?.type).to.equal('startOrder') @@ -267,264 +231,70 @@ describe('********** OrderDatabase CRUD', () => { }) it('delete order', async () => { + const before = await database.order.retrieve('order1.0') + expect(before?.consumer).to.equal('0x1235') + expect(before?.timestamp).to.equal(1678593730) const result = await database.order.delete('order1.0') - expect(result?.id).to.equal('order1.0') - expect(result?.consumer).to.equal('0x1235') - expect(result?.payer).to.equal('0x4567') - expect(result?.type).to.equal('startOrder') - expect(result?.timestamp).to.equal(1678593730) - }) -}) - -describe('********** Typesense OrderDatabase CRUD', () => { - let database: AbstractOrderDatabase - - before(async () => { - database = await DatabaseFactory.createOrderDatabase(typesenseConfig) - }) - - it('creates an order in Typesense', async () => { - const result = await database.create( - 'orderTypesense1', - 'startOrder', - 1678593728, - '0x1234', - '0x4567', - '0x1111', - '0x2', - 'did:' - ) - expect(result?.id).to.equal('orderTypesense1') - expect(result?.consumer).to.equal('0x1234') - }) - - it('retrieves an order from Typesense', async () => { - const result = await database.retrieve('orderTypesense1') - expect(result?.id).to.equal('orderTypesense1') - expect(result?.consumer).to.equal('0x1234') - }) - - it('updates an order in Typesense', async () => { - const result = await database.update( - 'orderTypesense1', - 'startOrder', - 1678593730, - '0x1235', - '0x4567' - ) - expect(result?.consumer).to.equal('0x1235') - }) - - it('deletes an order from Typesense', async () => { - const result = await database.delete('orderTypesense1') - expect(result?.id).to.equal('orderTypesense1') + expect(result).to.deep.equal({ id: 'order1.0' }) + expect(await database.order.retrieve('order1.0')).to.equal(null) }) }) -describe('********** Elasticsearch OrderDatabase CRUD', () => { - let database: AbstractOrderDatabase +describeOnElasticsearch('********** DDO state and order search', () => { + let database: Database + let client: Client + const consumer = Wallet.createRandom().address + const payer = Wallet.createRandom().address before(async () => { - database = await DatabaseFactory.createOrderDatabase(elasticConfig) - }) - - it('creates an order in Elasticsearch', async () => { - const result = await database.create( - 'orderElastic1', - 'startOrder', - 1678593728, - '0x1234', - '0x4567', - '0x1111', - '0x1', - 'did:' - ) - expect(result?.orderId).to.equal('orderElastic1') - expect(result?.consumer).to.equal('0x1234') - }) - - it('retrieves an order from Elasticsearch', async () => { - const result = await database.retrieve('orderElastic1') - expect(result?.orderId).to.equal('orderElastic1') - expect(result?.consumer).to.equal('0x1234') - expect(result?.payer).to.equal('0x4567') - expect(result?.type).to.equal('startOrder') - }) - - it('updates an order in Elasticsearch', async () => { - const result = await database.update( - 'orderElastic1', - 'startOrder', - 1678593730, - '0x1235', - '0x4567', - '0x1111' - ) - expect(result?.consumer).to.equal('0x1235') - }) - - it('deletes an order from Elasticsearch', async () => { - const result = await database.delete('orderElastic1') - expect(result?.id).to.equal('orderElastic1') - }) -}) - -describe('********** DdoStateQuery', () => { - it('should build Typesense query for did', async () => { - const query = (await DatabaseFactory.createDdoStateQuery(typesenseConfig)).buildQuery( - 'did:op:abc123' - ) - expect(query.q).to.equal('did:op:abc123') - expect(query.query_by).to.equal('did') - }) - - it('should build Typesense query for nft', async () => { - const query = (await DatabaseFactory.createDdoStateQuery(typesenseConfig)).buildQuery( - undefined, - 'nft:op:abc123' - ) - expect(query.q).to.equal('nft:op:abc123') - expect(query.query_by).to.equal('nft') - }) - - it('should build Typesense query for txId', async () => { - const query = (await DatabaseFactory.createDdoStateQuery(typesenseConfig)).buildQuery( - undefined, - undefined, - 'txId123' - ) - expect(query.q).to.equal('txId123') - expect(query.query_by).to.equal('txId') - }) - - it('should build Elasticsearch query for did', async () => { - const query = (await DatabaseFactory.createDdoStateQuery(elasticConfig)).buildQuery( - 'did:op:abc123' - ) - expect(query.term.did).to.equal('did:op:abc123') - }) - - it('should build Elasticsearch query for nft', async () => { - const query = (await DatabaseFactory.createDdoStateQuery(elasticConfig)).buildQuery( - undefined, - 'nft:op:abc123' - ) - expect(query.term.nft).to.equal('nft:op:abc123') - }) - - it('should build Elasticsearch query for txId', async () => { - const query = (await DatabaseFactory.createDdoStateQuery(elasticConfig)).buildQuery( - undefined, - undefined, - 'txId123' - ) - expect(query.term.txId).to.equal('txId123') - }) -}) - -describe('********** MetadataQuery', () => { - it('should return a Typesense query when DB is Typesense and a Typesense query is passed', async () => { - const typesenseQuery = { - q: '*', - filter_by: - 'author:=Ocean && metadata.type:=[dataset,algorithm] && purgatory_state:!=true', - num_hits: 10, - start: 0, - sort_by: 'name:asc' + database = await Database.init(elasticConfig) + client = await createElasticsearchClientWithRetry(elasticConfig) + }) + + it('finds a DDO state by did, nft or txId', async () => { + await database.ddoState.create(8996, 'did:op:state-query', '0xNftQuery', '0xTxQuery') + await client.indices.refresh({ index: 'ddo_state' }) + for (const key of [ + { did: 'did:op:state-query' }, + { nft: '0xNftQuery' }, + { txId: '0xTxQuery' } + ]) { + const found = await database.ddoState.find(key) + expect(found.map((state: any) => state.id)).to.deep.equal(['did:op:state-query']) } - - const query = (await DatabaseFactory.createMetadataQuery(typesenseConfig)).buildQuery( - typesenseQuery - ) - expect(query.q).to.equal('*') - expect(query.num_hits).to.equal(10) - expect(query.start).to.equal(0) - expect(query.filter_by).to.equal( - 'author:=Ocean && metadata.type:=[dataset,algorithm] && purgatory_state:!=true' - ) - expect(query.sort_by).to.equal('name:asc') - }) - - it('should convert an Elasticsearch query to a Typesense query when DB is Typesense', async () => { - const searchQuery: SearchQuery = { - query: { - bool: { - filter: [ - { term: { author: 'Ocean' } }, - { terms: { 'metadata.type': ['dataset', 'algorithm'] } } - ], - must_not: [{ term: { purgatory_state: true } }] - } - }, - size: 10, - from: 0, - sort: { name: 'asc' } - } - - const query = (await DatabaseFactory.createMetadataQuery(typesenseConfig)).buildQuery( - searchQuery - ) - expect(query.q).to.equal('*') - expect(query.num_hits).to.equal(10) - expect(query.start).to.equal(0) - expect(query.filter_by).to.contain('author:=Ocean') - expect(query.filter_by).to.contain('metadata.type:=[dataset,algorithm]') - expect(query.filter_by).to.contain('purgatory_state:!=true') - expect(query.sort_by).to.equal('name:asc') - }) - - it('should convert a Typesense query to an Elasticsearch query when DB is Elasticsearch', async () => { - const typesenseQuery = { - q: '*', - filter_by: - 'author:=Ocean && metadata.type:=[dataset,algorithm] && purgatory_state:!=true', - num_hits: 10, - start: 0, - sort_by: 'name:asc' - } - - const query = (await DatabaseFactory.createMetadataQuery(elasticConfig)).buildQuery( - typesenseQuery + await database.ddoState.delete('did:op:state-query') + }) + + it('finds the orders of a consumer or a payer in any letter case, newest first', async () => { + const create = (orderId: string, timestamp: number) => + database.order.create( + orderId, + 'startOrder', + timestamp, + consumer, + payer, + '0xDatatoken', + '0xNft', + 'did:op:order-query' + ) + await create('0xorder-query-1', 100) + await create('0xorder-query-2', 200) + await client.indices.refresh({ index: 'order' }) + const all = { offset: 0, limit: 10 } + const byConsumer = await database.order.find( + { consumer: consumer.toLowerCase() }, + all ) - expect(query.size).to.equal(10) - expect(query.from).to.equal(0) - expect(query.query.bool.filter[0].term.author).to.equal('Ocean') - expect(query.query.bool.filter[1].terms['metadata.type']).to.eql([ - 'dataset', - 'algorithm' + expect(byConsumer.results.map((order: any) => order.orderId)).to.deep.equal([ + '0xorder-query-2', + '0xorder-query-1' ]) - expect(query.query.bool.must_not[0].term.purgatory_state).to.equal('true') - expect(query.sort[0].name.order).to.equal('asc') - }) - - it('should return an Elasticsearch query when DB is Elasticsearch and an Elasticsearch query is passed', async () => { - const searchQuery: SearchQuery = { - query: { - bool: { - filter: [ - { term: { author: 'Ocean' } }, - { terms: { 'metadata.type': ['dataset', 'algorithm'] } } - ], - must_not: [{ term: { purgatory_state: true } }] - } - }, - size: 10, - from: 0, - sort: { name: 'asc' } - } - - const query = (await DatabaseFactory.createMetadataQuery(elasticConfig)).buildQuery( - searchQuery - ) - - expect(query.size).to.equal(10) - expect(query.from).to.equal(0) - expect(query.query.bool.filter[0].term.author).to.equal('Ocean') - expect(query.query.bool.filter[1].terms['metadata.type']).to.eql([ - 'dataset', - 'algorithm' + expect(byConsumer.totalResults).to.equal(2) + const byPayer = await database.order.find({ payer }, { offset: 1, limit: 1 }) + expect(byPayer.results.map((order: any) => order.orderId)).to.deep.equal([ + '0xorder-query-1' ]) - expect(query.query.bool.must_not[0].term.purgatory_state).to.equal(true) - expect(query.sort.name).to.equal('asc') + await database.order.delete('0xorder-query-1') + await database.order.delete('0xorder-query-2') }) }) diff --git a/src/test/integration/elasticsearch.test.ts b/src/test/integration/elasticsearch.test.ts index 672772d86..3cfc3b030 100644 --- a/src/test/integration/elasticsearch.test.ts +++ b/src/test/integration/elasticsearch.test.ts @@ -1,40 +1,48 @@ +import { describeOnElasticsearch } from '../utils/utils.js' import { ddo } from '../data/ddo.js' import { expect } from 'chai' import { Database } from '../../components/database/index.js' +import type { OceanNodeDBConfig } from '../../@types/OceanNode.js' import { ElasticsearchDdoDatabase, ElasticsearchDdoStateDatabase, ElasticsearchIndexerDatabase, ElasticsearchLogDatabase, + ElasticsearchNonceDatabase, ElasticsearchOrderDatabase } from '../../components/database/ElasticSearchDatabase.js' -import { DB_TYPES } from '../../utils/index.js' -import { SQLLiteNonceDatabase } from '../../components/database/SQLLiteNonceDatabase.js' -const dbConfig = { +const dbConfig: OceanNodeDBConfig = { + dbType: 'elasticsearch', url: 'http://localhost:9200', - dbType: DB_TYPES.ELASTIC_SEARCH, username: 'elastic', password: 'changeme' } -const elasticsearch: Database = await Database.init(dbConfig) +let elasticsearch: Database +// the suites of this file share one connection; it is made when the first suite runs +const connect = async () => { + elasticsearch ??= await Database.init(dbConfig) +} + +describeOnElasticsearch('********** Elastic Search', () => { + before(connect) -describe('********** Elastic Search', () => { it('Get instances of Elastic Search', () => { expect(elasticsearch.ddo).to.be.instanceOf(ElasticsearchDdoDatabase) expect(elasticsearch.indexer).to.be.instanceOf(ElasticsearchIndexerDatabase) expect(elasticsearch.ddoState).to.be.instanceOf(ElasticsearchDdoStateDatabase) expect(elasticsearch.order).to.be.instanceOf(ElasticsearchOrderDatabase) - expect(elasticsearch.nonce).to.be.instanceOf(SQLLiteNonceDatabase) + expect(elasticsearch.nonce).to.be.instanceOf(ElasticsearchNonceDatabase) expect(elasticsearch.logs).to.be.instanceOf(ElasticsearchLogDatabase) }) }) -describe('********** Elastic Search DDO collections', () => { +describeOnElasticsearch('********** Elastic Search DDO collections', () => { + before(connect) + it('create document in ddo collection', async () => { const result = await elasticsearch.ddo.create(ddo) - expect(result.result).to.equal('created') - expect(result._id).to.equal(ddo.id) + expect(result.id).to.equal(ddo.id) // expect(result.metadata).to.not.be.an('undefined') // expect(result.metadata.name).to.be.equal(ddo.metadata.name) }) @@ -51,13 +59,11 @@ describe('********** Elastic Search DDO collections', () => { const updatedData = ddo updatedData.metadata.name = newMetadataName const result = await elasticsearch.ddo.update(updatedData) - expect(result.result).to.equal('updated') - expect(result._id).to.equal(updatedData.id) + expect(result.id).to.equal(updatedData.id) + expect(result.metadata.name).to.equal(newMetadataName) }) it('delete document in ddo collection', async () => { - const result = await elasticsearch.ddo.delete(ddo.id) - expect(result.result).to.equal('deleted') - expect(result._id).to.equal(ddo.id) + expect(await elasticsearch.ddo.delete(ddo.id)).to.deep.equal({ id: ddo.id }) }) }) diff --git a/src/test/integration/escrow.test.ts b/src/test/integration/escrow.test.ts index 2c122add3..3300eef3a 100644 --- a/src/test/integration/escrow.test.ts +++ b/src/test/integration/escrow.test.ts @@ -233,13 +233,36 @@ describe('Indexer stores Escrow contract events', () => { if (!escrowAddress || !paymentToken) this.skip() this.timeout(DEFAULT_TEST_TIMEOUT * 3) - // reLock adjusts the amount/expiry of the lock created in the previous test - // for the same jobId, preserving the original creation timestamp. The new - // expiry is a duration-from-now capped by (creationTimestamp + maxLockSeconds), - // so use a value well below `expiry` (== maxLockSeconds) to leave headroom for - // the wall-clock time spent waiting on the Lock event to be indexed above. + // reLock adjusts the amount/expiry of the lock created in the previous test for the same + // jobId. The new expiry is a duration-from-now capped by (lock.startTime + maxLockSeconds). + // Deriving that duration from the real on-chain lock + authorization — instead of a blind + // `expiry/2` relative to the createLock parameter — keeps the call deterministic: the old + // approach intermittently overshot the ceiling and reverted at estimateGas when the local + // chain's block.timestamp drifted ahead of wall clock between createLock and reLock. const newLockAmount = parseUnits('20', 18) - const reLockExpiry = Math.floor(expiry / 2) + + const locks = await escrowContract.getLocks(paymentToken, payerAddress, payeeAddress) + const lock = Array.from(locks).find( + (l: any) => l.jobId.toString() === jobId.toString() + ) as any + assert(lock, 'createLock should have left an on-chain lock for this jobId') + + const auths = await escrowContract.getAuthorizations( + paymentToken, + payerAddress, + payeeAddress + ) + const maxLockSeconds = Number((auths[0] as any).maxLockSeconds) + const startTime = Number(lock.startTime) + const nowTs = Number((await provider.getBlock('latest'))?.timestamp) + // Width of the still-valid window: the new expiry must land in (now, startTime + maxLockSeconds]. + const remainingWindow = startTime + maxLockSeconds - nowTs + // If the whole window has already elapsed on-chain there is nothing valid to reLock into — + // skip rather than flake (shouldn't happen with maxLockSeconds=7200 and seconds between txs). + if (!Number.isFinite(remainingWindow) || remainingWindow <= 2) this.skip() + // Sit in the middle of the window: a valid duration-from-now, comfortably below the ceiling. + const reLockExpiry = Math.max(1, Math.floor(remainingWindow / 2)) + const tx = await escrowContract .connect(publisherAccount) // Escrow v2: + jobType (0 = NONE) + subsidyProviders ([] = plain payer-funded lock) diff --git a/src/test/integration/imageCleanup.test.ts b/src/test/integration/imageCleanup.test.ts index 3c8d06302..c3ce52ec9 100644 --- a/src/test/integration/imageCleanup.test.ts +++ b/src/test/integration/imageCleanup.test.ts @@ -1,7 +1,7 @@ import { expect, assert } from 'chai' -import { SQLiteCompute } from '../../components/database/sqliteCompute.js' +import sinon from 'sinon' import { C2DDatabase } from '../../components/database/C2DDatabase.js' -import { typesenseSchemas } from '../../components/database/TypesenseSchemas.js' +import { Database } from '../../components/database/index.js' import { getConfiguration } from '../../utils/config.js' import { buildEnvOverrideConfig, @@ -18,11 +18,14 @@ import { KeyManager } from '../../components/KeyManager/index.js' import { C2DClusterInfo } from '../../@types/C2D/C2D.js' import Dockerode from 'dockerode' +const MS_PER_DAY = 24 * 60 * 60 * 1000 +// a negative retention puts the cutoff in the future: every tracked image is listed +const ALL_IMAGES = -1 + describe('********** Docker Image Cleanup Integration Tests', () => { let envOverrides: OverrideEnvConfig[] let config: OceanNodeConfig let db: C2DDatabase - let sqliteProvider: SQLiteCompute let dockerEngine: C2DEngineDocker let docker: Dockerode @@ -67,13 +70,25 @@ describe('********** Docker Image Cleanup Integration Tests', () => { ) envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) config = await getConfiguration(true) - db = await new C2DDatabase(config.dbConfig, typesenseSchemas.c2dSchemas) - sqliteProvider = (db as any).provider as SQLiteCompute + db = (await Database.init(config.dbConfig)).c2d // Initialize Docker connection for testing docker = new Dockerode() }) + // records the image as last used `days` ago: only Date is faked, timers stay real + async function updateImageDaysAgo(image: string, days: number): Promise { + const clock = sinon.useFakeTimers({ + now: Date.now() - days * MS_PER_DAY, + toFake: ['Date'] + }) + try { + await db.updateImage(image) + } finally { + clock.restore() + } + } + after(async () => { await tearDownEnvironment(envOverrides) if (dockerEngine) { @@ -82,82 +97,43 @@ describe('********** Docker Image Cleanup Integration Tests', () => { }) describe('Image Tracking Database Methods', () => { - it('should create docker_images table', async () => { - await sqliteProvider.createImageTable() - // If no error is thrown, table creation succeeded - assert(true, 'Table creation should succeed') - }) - it('should update image usage timestamp', async () => { const testImage = 'test-image:latest' - await sqliteProvider.updateImage(testImage) - - // Verify image was recorded by querying directly via the SqliteClient handle. - // getOldImages(0) looks for images older than now, so we query the DB directly. - const sqliteClient = (sqliteProvider as any).db - const imageRecord = sqliteClient.get( - 'SELECT image, lastUsedTimestamp FROM docker_images WHERE image = ?', - [testImage] - ) + await db.updateImage(testImage) - assert(imageRecord, 'Image should be recorded in database') - expect(imageRecord.image).to.equal(testImage) - expect(imageRecord.lastUsedTimestamp).to.be.a('number') - const currentTimestamp = Math.floor(Date.now() / 1000) - expect(imageRecord.lastUsedTimestamp).to.be.at.least(currentTimestamp - 1) + expect(await db.getOldImages(ALL_IMAGES)).to.include(testImage) + expect(await db.getOldImages(7)).to.not.include(testImage) }) it('should update existing image timestamp', async () => { const testImage = 'test-image-update:latest' - const firstTimestamp = Math.floor(Date.now() / 1000) - - // Insert image first time - await sqliteProvider.updateImage(testImage) - // Wait a bit to ensure timestamp difference - await new Promise((resolve) => setTimeout(resolve, 1100)) + await updateImageDaysAgo(testImage, 8) + expect(await db.getOldImages(7)).to.include(testImage) - // Update image again - timestamp should be newer - await sqliteProvider.updateImage(testImage) - - // Verify image exists with updated timestamp - const sqliteClient = (sqliteProvider as any).db - const imageRecord = sqliteClient.get( - 'SELECT image, lastUsedTimestamp FROM docker_images WHERE image = ?', - [testImage] - ) - - assert(imageRecord, 'Image should be recorded in database') - expect(imageRecord.image).to.equal(testImage) - expect(imageRecord.lastUsedTimestamp).to.be.greaterThan(firstTimestamp) + // using the image again makes it recent + await db.updateImage(testImage) + expect(await db.getOldImages(7)).to.not.include(testImage) }) it('should return old images based on retention days', async () => { const recentImage = 'recent-image:latest' const oldImage = 'old-image:latest' - // Update recent image - await sqliteProvider.updateImage(recentImage) - - // Manually insert an old image by directly updating the database - const oldTimestamp = Math.floor(Date.now() / 1000) - 8 * 24 * 60 * 60 // 8 days ago - const sqliteClient = (sqliteProvider as any).db - sqliteClient.run( - 'INSERT OR REPLACE INTO docker_images (image, lastUsedTimestamp) VALUES (?, ?)', - [oldImage, oldTimestamp] - ) + await db.updateImage(recentImage) + await updateImageDaysAgo(oldImage, 8) // Get images older than 7 days - const oldImages = await sqliteProvider.getOldImages(7) + const oldImages = await db.getOldImages(7) expect(oldImages).to.include(oldImage) expect(oldImages).to.not.include(recentImage) }) it('should return empty array when no old images exist', async () => { const recentImage = 'very-recent-image:latest' - await sqliteProvider.updateImage(recentImage) + await db.updateImage(recentImage) - const oldImages = await sqliteProvider.getOldImages(30) // 30 days retention + const oldImages = await db.getOldImages(30) // 30 days retention expect(oldImages).to.not.include(recentImage) }) }) @@ -189,15 +165,10 @@ describe('********** Docker Image Cleanup Integration Tests', () => { // Call updateImageUsage directly (using private method access for testing) await (dockerEngine as any).updateImageUsage(testImage) - // Verify image was recorded in database - const sqliteClient = (sqliteProvider as any).db - const imageRecord = sqliteClient.get( - 'SELECT image, lastUsedTimestamp FROM docker_images WHERE image = ?', - [testImage] - ) - - assert(imageRecord, 'Image should be recorded in database after updateImageUsage') - expect(imageRecord.image).to.equal(testImage) + expect( + await db.getOldImages(ALL_IMAGES), + 'Image should be recorded in database after updateImageUsage' + ).to.include(testImage) }) it('should start image cleanup timer on engine start', () => { @@ -217,7 +188,7 @@ describe('********** Docker Image Cleanup Integration Tests', () => { const nonExistentImage = 'non-existent-image:999999' // Manually insert into database - await sqliteProvider.updateImage(nonExistentImage) + await db.updateImage(nonExistentImage) // Try to clean it up - should not throw error try { @@ -261,12 +232,7 @@ describe('********** Docker Image Cleanup Integration Tests', () => { } // Track the image with old timestamp (8 days ago) - const oldTimestamp = Math.floor(Date.now() / 1000) - 8 * 24 * 60 * 60 - const sqliteClient = (sqliteProvider as any).db - sqliteClient.run( - 'INSERT OR REPLACE INTO docker_images (image, lastUsedTimestamp) VALUES (?, ?)', - [testImageName, oldTimestamp] - ) + await updateImageDaysAgo(testImageName, 8) // Verify image exists before cleanup const imagesBefore = await docker.listImages() diff --git a/src/test/integration/indexer.test.ts b/src/test/integration/indexer.test.ts index 4709148a7..b1c2b02a5 100644 --- a/src/test/integration/indexer.test.ts +++ b/src/test/integration/indexer.test.ts @@ -15,7 +15,6 @@ import ERC721Factory from '@oceanprotocol/contracts/artifacts/contracts/ERC721Fa import ERC721Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC721Template.sol/ERC721Template.json' with { type: 'json' } import ERC20Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20TemplateEnterprise.sol/ERC20TemplateEnterprise.json' with { type: 'json' } import { Database } from '../../components/database/index.js' -import { DatabaseFactory } from '../../components/database/DatabaseFactory.js' import { OceanIndexer } from '../../components/Indexer/index.js' import { RPCS } from '../../@types/blockchain.js' import { getEventFromTx, streamToObject } from '../../utils/util.js' @@ -278,10 +277,8 @@ describe('********** Indexer stores a new metadata events and orders.', it('should find the state of the ddo using query ddo state handler', async function () { const queryDdoStateHandler = new QueryDdoStateHandler(oceanNode) - const config = await getConfiguration(true) - const queryStrategy = await DatabaseFactory.createDdoStateQuery(config.dbConfig) const queryDdoState: QueryCommand = { - query: queryStrategy.buildQuery(resolvedDDO.id), + query: { did: resolvedDDO.id }, command: PROTOCOL_COMMANDS.QUERY } const response = await queryDdoStateHandler.handle(queryDdoState) @@ -496,7 +493,7 @@ describe('********** Indexer stores a new metadata events and orders.', const resultOrder = await database.order.retrieve(orderTxId) if (resultOrder) { if (resultOrder.id) { - // typesense response + // response keyed by id expect(resultOrder.id).to.equal(orderTxId) } else if (resultOrder.orderId) { // elastic search response @@ -591,7 +588,7 @@ describe('********** Indexer stores a new metadata events and orders.', const resultOrder = await database.order.retrieve(reuseOrderTxId) if (resultOrder) { if (resultOrder.id) { - // typesense + // response keyed by id expect(resultOrder.id).to.equal(reuseOrderTxId) } else if (resultOrder.orderId) { // elastic diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts index 82b32e149..ce7c2dd2f 100644 --- a/src/test/integration/logs.test.ts +++ b/src/test/integration/logs.test.ts @@ -146,7 +146,7 @@ describe('********** LogDatabase CRUD', () => { const startTime = new Date(Date.now() - 2500) // 2.5 seconds ago const endTime = new Date() // current time - // we cannot predict the amount of logs written on DB (Typesense adds tons on its own), so we need: + // we cannot predict the amount of logs written on DB (the DB adds tons on its own), so we need: // 1 ) set a smaller interval // 2 ) retrieve a bigger number of logs // 3 ) filter the appropriate message diff --git a/src/test/integration/maintenanceMode.test.ts b/src/test/integration/maintenanceMode.test.ts index 1a472fcd2..77e1577cd 100644 --- a/src/test/integration/maintenanceMode.test.ts +++ b/src/test/integration/maintenanceMode.test.ts @@ -28,7 +28,7 @@ import { P2PCommandResponse } from '../../@types/OceanNode.js' /** * Drain-scenario integration coverage that does NOT require Barge: the maintenance guards * short-circuit before any chain / Docker / escrow work, so this exercises the real handlers, - * the real SQLite-config persistence and the real status handler end-to-end. + * the real config-DB persistence and the real status handler end-to-end. * * NOT covered here (requires a running Barge chain + Docker): that a compute job / service * STARTED before enabling maintenance still returns status/results/logs and can be stopped. @@ -62,7 +62,7 @@ describe('Maintenance mode drain scenario (integration)', () => { envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) delete process.env.MAINTENANCE_MODE const config = await getConfiguration(true) - db = await Database.init({ url: 'http://localhost:9999', dbType: 'none' } as any) + db = await Database.init(config.dbConfig) const keyManager = new KeyManager(config) const blockchainRegistry = new BlockchainRegistry(keyManager, config) oceanNode = OceanNode.getInstance( @@ -145,7 +145,7 @@ describe('Maintenance mode drain scenario (integration)', () => { const status = await body(resp) expect(status.maintenance).to.not.equal(undefined) expect(status.maintenance.enabled).to.equal(true) - // drain-visibility count is present as a number (the shared SQLite C2D DB may carry + // drain-visibility count is present as a number (the shared C2D DB may carry // leftover rows from other suites, so we assert the field exists rather than a fixed value) expect(status.maintenance.runningComputeJobs).to.be.a('number') }) @@ -182,7 +182,7 @@ describe('Maintenance mode drain scenario (integration)', () => { it('persists the flag across a simulated restart (reload from DB)', async () => { await oceanNode.setMaintenanceMode(true) - // Wipe the in-memory cache and reload from the SQLite config DB, as startup does. + // Wipe the in-memory cache and reload from the config DB, as startup does. ;(oceanNode as any).maintenanceMode = false await oceanNode.loadMaintenanceMode() expect(oceanNode.isInMaintenance()).to.equal(true) diff --git a/src/test/integration/nonce.test.ts b/src/test/integration/nonce.test.ts deleted file mode 100644 index 575089954..000000000 --- a/src/test/integration/nonce.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { expect, assert } from 'chai' -import { ZeroAddress } from 'ethers' -import { nonceSchema } from '../data/nonceSchema.js' -import { Typesense, convertTypesenseConfig } from '../../components/database/typesense.js' - -describe('********** Nonce tests', () => { - let typesense: Typesense - let error: Error - - before(() => { - const url = 'http://localhost:8108/?apiKey=xyz' - typesense = new Typesense(convertTypesenseConfig(url)) - }) - - it('instance Typesense', () => { - expect(typesense).to.be.instanceOf(Typesense) - }) - - it('create nonce collection', async () => { - let result - try { - result = await typesense.collections(nonceSchema.name).retrieve() - } catch (error) { - result = await typesense.collections().create(nonceSchema) - } - expect(result.enable_nested_fields).to.equal(true) - expect(result.fields).to.not.be.an('undefined') - expect(result.name).to.be.equal(nonceSchema.name) - assert(result.num_documents >= 0, 'num_documents is not a valid number') - }) - - it('should get nonce (1)', async () => { - await typesense.collections(nonceSchema.name).documents().create({ - id: '0x4cc9DBfc4bEeA8c986c61DAABB350C2eC55e29d1', - nonce: 1 - }) - const document = await typesense - .collections(nonceSchema.name) - .documents() - .retrieve('0x4cc9DBfc4bEeA8c986c61DAABB350C2eC55e29d1') - expect(document.nonce).to.be.equal(1) - }) - - it('should throw error for retrieving unexistent address', async () => { - try { - await typesense.collections(nonceSchema.name).documents().retrieve(ZeroAddress) - } catch (err) { - error = err - } - expect(error.message).to.eql( - 'Could not find a document with id: 0x0000000000000000000000000000000000000000' - ) - }) -}) diff --git a/src/test/integration/persistentStorage.test.ts b/src/test/integration/persistentStorage.test.ts index 36738365b..d26d7cf8b 100644 --- a/src/test/integration/persistentStorage.test.ts +++ b/src/test/integration/persistentStorage.test.ts @@ -45,6 +45,7 @@ import { DEVELOPMENT_CHAIN_ID } from '../../utils/address.js' import { deployAndGetAccessListConfig } from '../utils/contracts.js' import { OceanNodeConfig, OceanNodeStatus } from '../../@types/OceanNode.js' import { KeyManager } from '../../components/KeyManager/index.js' +import { createElasticsearchClientWithRetry } from '../../components/database/ElasticsearchConfigHelper.js' describe('********** Persistent storage handlers (integration)', function () { this.timeout(DEFAULT_TEST_TIMEOUT) @@ -124,6 +125,18 @@ describe('********** Persistent storage handlers (integration)', functio consumer = (await provider.getSigner(1)) as Signer }) + // an Elasticsearch index outlives the run: the buckets of an earlier run must not count + beforeEach(async () => { + if (config.dbConfig.dbType !== 'elasticsearch') return + const client = await createElasticsearchClientWithRetry(config.dbConfig) + await client.deleteByQuery({ + index: 'buckets', + query: { match_all: {} }, + refresh: true, + conflicts: 'proceed' + }) + }) + after(async () => { await oceanNode.tearDownAll() await tearDownEnvironment(previousConfiguration) @@ -157,7 +170,7 @@ describe('********** Persistent storage handlers (integration)', functio }) } as unknown as OceanNode - const backend = new PersistentStorageLocalFS(fakeNode) + const backend = new PersistentStorageLocalFS(fakeNode, database.buckets) try { const ownerAddress = await consumer.getAddress() const { bucketId } = await backend.createNewBucket([], ownerAddress) diff --git a/src/test/integration/typesense.test.ts b/src/test/integration/typesense.test.ts deleted file mode 100644 index b6ee12c4c..000000000 --- a/src/test/integration/typesense.test.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { - Typesense, - convertTypesenseConfig, - TypesenseCollections -} from '../../components/database/typesense.js' -import { ddoSchema } from '../data/ddoSchema.js' -import { ddo } from '../data/ddo.js' -import { expect } from 'chai' -import { TypesenseSearchParams } from '../../@types/index.js' - -describe('********** Typesense', () => { - let typesense: Typesense - - before(() => { - const url = 'http://localhost:8108/?apiKey=xyz' - typesense = new Typesense(convertTypesenseConfig(url)) - }) - - it('instance Typesense', () => { - expect(typesense).to.be.instanceOf(Typesense) - }) - - it('instance TypesenseCollections', () => { - const result = typesense.collections() - expect(result).to.be.instanceOf(TypesenseCollections) - }) -}) - -describe('********** Typesense collections', () => { - let typesense: Typesense - - before(() => { - const url = 'http://localhost:8108/?apiKey=xyz' - typesense = new Typesense(convertTypesenseConfig(url)) - }) - - it('create ddo collection', async () => { - const result = await typesense.collections().create(ddoSchema) - expect(result.enable_nested_fields).to.equal(true) - expect(result.fields).to.not.be.an('undefined') - expect(result.name).to.be.equal(ddoSchema.name) - expect(result.num_documents).to.equal(0) - }) - - it('retrieve collections', async () => { - const result = await typesense.collections().retrieve() - const collection = result[0] - expect(collection.enable_nested_fields).to.equal(true) - expect(collection.fields).to.not.be.an('undefined') - expect(collection.name).to.be.equal(ddoSchema.name) - expect(collection.num_documents).to.equal(0) - }) - - it('retrieve ddo collection', async () => { - const result = await typesense.collections(ddoSchema.name).retrieve() - expect(result.enable_nested_fields).to.equal(true) - expect(result.fields).to.not.be.an('undefined') - expect(result.name).to.be.equal(ddoSchema.name) - expect(result.num_documents).to.equal(0) - }) - - it('update ddo collection', async () => { - const result = await typesense.collections(ddoSchema.name).update({ - fields: [{ name: 'nftAddress', drop: true }] - }) - expect(result.fields).to.not.be.an('undefined') - expect(result.fields[0].drop).to.equal(true) - expect(result.fields[0].name).to.be.equal('nftAddress') - }) - - it('delete ddo collection', async () => { - const result = await typesense.collections(ddoSchema.name).delete() - expect(result.enable_nested_fields).to.equal(true) - expect(result.fields).to.not.be.an('undefined') - expect(result.name).to.be.equal(ddoSchema.name) - }) -}) - -describe('********** Typesense documents', () => { - let typesense: Typesense - - before(() => { - const url = 'http://localhost:8108/?apiKey=xyz' - typesense = new Typesense(convertTypesenseConfig(url)) - }) - - it('create ddo collection', async () => { - const result = await typesense.collections().create(ddoSchema) - expect(result.enable_nested_fields).to.equal(true) - expect(result.fields).to.not.be.an('undefined') - expect(result.name).to.be.equal(ddoSchema.name) - expect(result.num_documents).to.equal(0) - }) - - it('create document in ddo collection', async () => { - const result = await typesense.collections(ddoSchema.name).documents().create(ddo) - expect(result.id).to.equal(ddo.id) - expect(result.metadata).to.not.be.an('undefined') - expect(result.metadata.name).to.be.equal(ddo.metadata.name) - }) - - it('retrieve document in ddo collection', async () => { - const result = await typesense - .collections(ddoSchema.name) - .documents() - .retrieve(ddo.id) - expect(result.id).to.equal(ddo.id) - expect(result.metadata).to.not.be.an('undefined') - expect(result.metadata.name).to.be.equal(ddo.metadata.name) - }) - - it('update document in ddo collection', async () => { - const newMetadataName = 'new metadata name' - const result = await typesense - .collections(ddoSchema.name) - .documents() - .update(ddo.id, { - metadata: { - name: newMetadataName - } - }) - expect(result.id).to.equal(ddo.id) - expect(result.metadata).to.not.be.an('undefined') - expect(result.metadata.name).to.be.equal(newMetadataName) - }) - - it('delete document in ddo collection', async () => { - const newMetadataName = 'new metadata name' - const result = await typesense.collections(ddoSchema.name).documents().delete(ddo.id) - expect(result.id).to.equal(ddo.id) - expect(result.metadata).to.not.be.an('undefined') - expect(result.metadata.name).to.be.equal(newMetadataName) - }) - - it('delete ddo collection', async () => { - const result = await typesense.collections(ddoSchema.name).delete() - expect(result.enable_nested_fields).to.equal(true) - expect(result.fields).to.not.be.an('undefined') - expect(result.name).to.equal(ddoSchema.name) - }) -}) - -describe('********** Typesense documents', () => { - let typesense: Typesense - - before(() => { - const url = 'http://localhost:8108/?apiKey=xyz' - typesense = new Typesense(convertTypesenseConfig(url)) - }) - - it('create ddo collection', async () => { - const result = await typesense.collections().create(ddoSchema) - expect(result.enable_nested_fields).to.equal(true) - expect(result.fields).to.not.be.an('undefined') - expect(result.name).to.be.equal(ddoSchema.name) - expect(result.num_documents).to.equal(0) - }) - - it('create document in ddo collection', async () => { - const result = await typesense.collections(ddoSchema.name).documents().create(ddo) - expect(result.id).to.equal(ddo.id) - expect(result.metadata).to.not.be.an('undefined') - expect(result.metadata.name).to.be.equal(ddo.metadata.name) - }) - it('search document in ddo collection', async () => { - const queryParams: TypesenseSearchParams = { - q: 'new metadata name', - query_by: 'metadata.name', - filter_by: 'chainId:=137', - sort_by: 'version:desc' - } - const result = await typesense - .collections(ddoSchema.name) - .documents() - .search(queryParams) - - expect(result.found).to.equal(1) - expect(result.hits[0]).to.not.be.an('undefined') - expect(result.hits[0].document).to.not.be.an('undefined') - }) - - it('delete ddo collection', async () => { - const result = await typesense.collections(ddoSchema.name).delete() - expect(result.enable_nested_fields).to.equal(true) - expect(result.fields).to.not.be.an('undefined') - expect(result.name).to.be.equal(ddoSchema.name) - }) -}) diff --git a/src/test/performance/util.js b/src/test/performance/util.js index 8a5092dd3..151e73b43 100644 --- a/src/test/performance/util.js +++ b/src/test/performance/util.js @@ -141,12 +141,11 @@ const SERVICE_ENDPOINTS = [ ['initialize', 'GET', '/api/services/initialize'], ['nonce', 'GET', '/api/services/nonce'], ['download', 'GET', '/api/services/download'], - // aquarius / assets / ddo - ['getDDO', 'GET', '/api/aquarius/assets/ddo/:did/:force?'], - ['getDDOMetadata', 'GET', '/api/aquarius/assets/metadata/:did/:force?'], - ['ddoMetadataQuery', 'POST', '/api/aquarius/assets/metadata/query'], - ['getDDOState', 'GET', '/api/aquarius/state/ddo'], - ['validateDDO', 'POST', '/api/aquarius/assets/ddo/validate'], + // ddo and query + ['getDDO', 'GET', '/api/ddo/:did/:force?'], + ['query', 'POST', '/api/query'], + ['getDDOState', 'GET', '/api/ddo/state'], + ['validateDDO', 'POST', '/api/ddo/validate'], // compute ['computeEnvironments', 'GET', '/api/services/computeEnvironments'], ['computeStart', 'POST', '/api/services/compute'], diff --git a/src/test/unit/auth/token.test.ts b/src/test/unit/auth/token.test.ts index f6c44d172..34bd106a7 100644 --- a/src/test/unit/auth/token.test.ts +++ b/src/test/unit/auth/token.test.ts @@ -1,24 +1,46 @@ +import { describeOnElasticsearch } from '../../utils/utils.js' import { OceanNodeConfig } from '../../../@types/OceanNode.js' import { getConfiguration } from '../../../utils/index.js' import { expect } from 'chai' import { HDNodeWallet, Wallet } from 'ethers' import { Auth } from '../../../components/Auth/index.js' import { AuthTokenDatabase } from '../../../components/database/AuthTokenDatabase.js' +import { Database } from '../../../components/database/index.js' import { OceanP2P } from '../../../components/P2P/index.js' import { Readable } from 'node:stream' import { createHashForSignature, safeSign } from '../../utils/signature.js' +import { Client } from '@elastic/elasticsearch' +import { + createElasticsearchClientWithRetry, + getOrNull +} from '../../../components/database/ElasticsearchConfigHelper.js' +import { create256Hash } from '../../../utils/crypt.js' -describe('Auth Token Tests', () => { +const AUTH_TOKENS_INDEX = 'auth_tokens' + +describeOnElasticsearch('Auth Token Tests', () => { let wallet: Wallet let authTokenDatabase: AuthTokenDatabase let config: OceanNodeConfig let auth: Auth + let client: Client before(async () => { config = await getConfiguration(true) - authTokenDatabase = await AuthTokenDatabase.create(config.dbConfig) + authTokenDatabase = (await Database.init(config.dbConfig)).authToken wallet = new Wallet(process.env.PRIVATE_KEY) auth = new Auth(authTokenDatabase, config) + client = await createElasticsearchClientWithRetry(config.dbConfig) + }) + + // every auth token write uses refresh: 'wait_for', so no index refresh is needed first + beforeEach(async () => { + await client.deleteByQuery({ + index: AUTH_TOKENS_INDEX, + query: { match_all: {} }, + refresh: true, + conflicts: 'proceed' + }) }) const getRandomNonce = () => { @@ -68,6 +90,48 @@ describe('Auth Token Tests', () => { const validationResult = await auth.validateToken(jwtToken) expect(validationResult).to.be.equal(null) }) + + it('stores the token as its sha256 hash, never in plaintext', async () => { + const jwtToken = await auth.getJWTToken(wallet.address, getRandomNonce(), Date.now()) + await auth.insertToken(wallet.address, jwtToken, Date.now() + 60_000, Date.now()) + + const doc = await client.get>({ + index: AUTH_TOKENS_INDEX, + id: create256Hash(jwtToken) + }) + expect(doc.found).to.be.equal(true) + expect(doc._source.token).to.be.equal(undefined) + expect(doc._source.address).to.be.equal(wallet.address) + }) + + it('never expires a token with validUntil null', async () => { + const jwtToken = await auth.getJWTToken(wallet.address, getRandomNonce(), Date.now()) + await authTokenDatabase.createToken(jwtToken, wallet.address, null, Date.now()) + + const entry = await auth.validateToken(jwtToken) + expect(entry).to.not.equal(null) + expect(entry.validUntil).to.be.equal(null) + expect(await authTokenDatabase.deleteExpiredTokens(Date.now() + 1e9)).to.be.equal(0) + }) + + it('deletes expired and invalidated tokens, keeps valid ones', async () => { + const now = Date.now() + const expired = await auth.getJWTToken(wallet.address, getRandomNonce(), now) + const invalidated = await auth.getJWTToken(wallet.address, getRandomNonce(), now) + const valid = await auth.getJWTToken(wallet.address, getRandomNonce(), now) + await auth.insertToken(wallet.address, expired, now - 1000, now - 2000) + await auth.insertToken(wallet.address, invalidated, now + 60_000, now) + await auth.insertToken(wallet.address, valid, now + 60_000, now) + await auth.invalidateToken(invalidated) + + expect(await authTokenDatabase.deleteExpiredTokens(Date.now())).to.be.equal(2) + + const exists = async (token: string) => + (await getOrNull(client, AUTH_TOKENS_INDEX, create256Hash(token))) !== null + expect(await exists(expired)).to.be.equal(false) + expect(await exists(invalidated)).to.be.equal(false) + expect(await exists(valid)).to.be.equal(true) + }) }) describe('Auth cross-node delegated validation', () => { @@ -131,7 +195,7 @@ describe('Auth cross-node delegated validation', () => { before(async () => { const baseConfig = await getConfiguration(true) config = { ...baseConfig, supportedNetworks: {} } - authTokenDatabase = await AuthTokenDatabase.create(baseConfig.dbConfig) + authTokenDatabase = (await Database.init(baseConfig.dbConfig)).authToken consumer = Wallet.createRandom() }) diff --git a/src/test/unit/compute.test.ts b/src/test/unit/compute.test.ts index 7be8b270f..ad2bb6217 100644 --- a/src/test/unit/compute.test.ts +++ b/src/test/unit/compute.test.ts @@ -1,7 +1,7 @@ import { C2DDatabase } from '../../components/database/C2DDatabase.js' +import { Database } from '../../components/database/index.js' // import { existsEnvironmentVariable, getConfiguration } from '../../utils/config.js' import { getConfiguration } from '../../utils/config.js' -import { typesenseSchemas } from '../../components/database/TypesenseSchemas.js' import { C2DStatusNumber, C2DStatusText, @@ -16,11 +16,6 @@ import { } from '../../@types/C2D/C2D.js' // import { computeAsset } from '../data/assets' import { assert, expect } from 'chai' -import { - convertArrayToString, - convertStringToArray, - STRING_SEPARATOR -} from '../../components/database/sqliteCompute.js' import os from 'os' import { buildEnvOverrideConfig, @@ -150,7 +145,7 @@ describe('Compute Jobs Database', () => { ) envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) config = await getConfiguration(true) - db = await new C2DDatabase(config.dbConfig, typesenseSchemas.c2dSchemas) + db = (await Database.init(config.dbConfig)).c2d }) it('should create a new C2D Job', async () => { @@ -330,19 +325,6 @@ describe('Compute Jobs Database', () => { expect(deleted === true, `Job ${jobId} was not deleted!`) }) - it('should convert array of strings to a string', () => { - const inputDID = ['did:op:1', 'did:op:2', 'did:op:3'] - const expectedStr = - 'did:op:1' + STRING_SEPARATOR + 'did:op:2' + STRING_SEPARATOR + 'did:op:3' - expect(convertArrayToString(inputDID)).to.equal(expectedStr) - }) - - it('should convert concatenated string to a string array', () => { - const expectedArray = ['did:op:1', 'did:op:2', 'did:op:3'] - const str = 'did:op:1' + STRING_SEPARATOR + 'did:op:2' + STRING_SEPARATOR + 'did:op:3' - expect(convertStringToArray(str)).to.deep.equal(expectedArray) - }) - it('should convert DBComputeJob to ComputeJob and omit internal DB data', () => { const source: any = completeDBComputeJob const output: ComputeJob = omitDBComputeFieldsFromComputeJob(source as DBComputeJob) @@ -2678,6 +2660,33 @@ describe('service start/restart Docker cleanup on failure', function () { expect(job.status).to.equal(ServiceStatusNumber.Error) }) + it('processJob fails a job in PullImage only when the stored job is still in PullImage', async function () { + engine.envs = [{ id: 'env-1' }] + engine.activeBuildAborts = new Map() + engine.recordJobFinished = sinon.stub() + engine.cleanupJob = sinon.stub().resolves() + engine.db.updateJob = sinon.stub().resolves(1) + const listed = { jobId: 'job-1', environment: 'env-1' } + + // the search result is older than the write of pullImage: the job has moved on + const stale = { ...listed, status: C2DStatusNumber.PullImage } + engine.db.getJob = sinon + .stub() + .resolves([{ ...listed, status: C2DStatusNumber.ConfiguringVolumes }]) + await engine.processJob(stale) + expect(engine.db.updateJob.called, 'must not touch a job that moved on').to.equal( + false + ) + expect(stale.status).to.equal(C2DStatusNumber.PullImage) + + // a real orphan: the stored job is still in PullImage and no pull runs + const orphan = { ...listed, status: C2DStatusNumber.PullImage } + engine.db.getJob = sinon.stub().resolves([{ ...orphan }]) + await engine.processJob(orphan) + expect(orphan.status).to.equal(C2DStatusNumber.PullImageFailed) + expect(engine.db.updateJob.calledOnce).to.equal(true) + }) + it('restartService removes the newly created network when createContainer fails', async function () { const existingJob = { serviceId: 'svc-3', @@ -2972,6 +2981,23 @@ describe('service start/restart Docker cleanup on failure', function () { ) ).to.equal(true) }) + + it('does not take the lifecycle lock again while its DB release is in flight', async function () { + let finishRelease: () => void + engine.db.releaseServiceLock = sinon.stub().returns( + new Promise((resolve) => { + finishRelease = resolve + }) + ) + expect(await engine.tryAcquireServiceLifecycleLock('svc-1')).to.equal(true) + const release = engine.releaseServiceLifecycleLock('svc-1') + // the release (which matches the holder) could delete a lock taken now + expect(await engine.tryAcquireServiceLifecycleLock('svc-1')).to.equal(false) + finishRelease() + await release + expect(await engine.tryAcquireServiceLifecycleLock('svc-1')).to.equal(true) + expect(engine.db.acquireServiceLock.callCount).to.equal(2) + }) }) // The admin-facing view of the metrics: one roll-up line per sampling interval, plus a diff --git a/src/test/unit/config.test.ts b/src/test/unit/config.test.ts index ba08b7538..864bfc08a 100644 --- a/src/test/unit/config.test.ts +++ b/src/test/unit/config.test.ts @@ -1,4 +1,7 @@ import { expect } from 'chai' +import os from 'os' +import fs from 'fs' +import path from 'path' import { getAddress } from 'ethers' import { OceanNodeConfig } from '../../@types/OceanNode.js' import { getConfiguration, loadConfigFromFile } from '../../utils/config.js' @@ -9,20 +12,27 @@ import { setupEnvironment, tearDownEnvironment } from '../utils/utils.js' -import { ENVIRONMENT_VARIABLES, JobType } from '../../utils/constants.js' +import { ENVIRONMENT_VARIABLES, EnvVariable, JobType } from '../../utils/constants.js' import { DEFAULT_DB_INIT_MAX_ATTEMPTS, DEFAULT_DB_INIT_MAX_RETRY_DELAY, DEFAULT_DB_INIT_RETRY_DELAY } from '../../utils/config/constants.js' +import { OceanNode } from '../../OceanNode.js' +import { KeyManager } from '../../components/KeyManager/index.js' let config: OceanNodeConfig describe('Should validate configuration from JSON', () => { let envOverrides: OverrideEnvConfig[] before(async () => { envOverrides = buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.DB_TYPE, ENVIRONMENT_VARIABLES.DB_URL], - ['typesense', 'http://localhost:8108/?apiKey=xyz'] + [ + ENVIRONMENT_VARIABLES.DB_TYPE, + ENVIRONMENT_VARIABLES.DB_URL, + ENVIRONMENT_VARIABLES.DB_USERNAME, + ENVIRONMENT_VARIABLES.DB_PASSWORD + ], + ['elasticsearch', 'http://127.0.0.1:9200', 'elastic', 'changeme'] ) envOverrides = await setupEnvironment(TEST_ENV_CONFIG_PATH, envOverrides) config = await getConfiguration(true) @@ -39,12 +49,9 @@ describe('Should validate configuration from JSON', () => { it('should have indexer', () => { expect(config.hasIndexer).to.be.equal(true) - expect(config.dbConfig).to.not.be.equal(null) - // it is exported in the env vars, so it should overwrite the config.json - expect(config.dbConfig.dbType).to.be.equal('typesense') - const configFile = loadConfigFromFile(process.env.CONFIG_PATH) - expect(config.dbConfig.dbType).to.not.be.equal(configFile.dbConfig.dbType) - expect(config.dbConfig.url).to.be.equal('http://localhost:8108/?apiKey=xyz') + // the database comes from the env vars: the config file has no database section + expect(config.dbConfig.dbType).to.be.equal('elasticsearch') + expect(config.dbConfig.url).to.be.equal('http://127.0.0.1:9200') }) it('should have HTTP', () => { @@ -67,24 +74,23 @@ describe('Should validate configuration from JSON', () => { expect(config.isBootstrap).to.be.equal(false) expect(config.validateUnsignedDDO).to.be.equal(true) }) - after(() => { + after(async () => { + // the database type must not stay selected for the tests that come after + await tearDownEnvironment(envOverrides) delete process.env.CONFIG_PATH delete process.env.PRIVATE_KEY }) }) describe('Should validate database init retry configuration', () => { - const DB_ENV_VARS = [ENVIRONMENT_VARIABLES.DB_TYPE, ENVIRONMENT_VARIABLES.DB_URL] - const DB_ENV_VALUES = ['typesense', 'http://localhost:8108/?apiKey=xyz'] - // returns the config built with the given DB_INIT_* values, or the thrown error async function configWith(values: { attempts?: string delay?: string maxDelay?: string }): Promise<{ config?: OceanNodeConfig; error?: Error }> { - const envVars = [...DB_ENV_VARS] - const envValues = [...DB_ENV_VALUES] + const envVars: EnvVariable[] = [] + const envValues: string[] = [] if (values.attempts !== undefined) { envVars.push(ENVIRONMENT_VARIABLES.DB_INIT_MAX_ATTEMPTS) envValues.push(values.attempts) @@ -161,6 +167,178 @@ describe('Should validate database init retry configuration', () => { }) }) +describe('Should select the database from DB_TYPE', () => { + const ELASTIC_URL = 'http://localhost:9200' + const POSTGRES_URL = 'postgresql://localhost:5432/ocean' + const TEST_CONFIG_FILE = '$HOME/config.json' + let configPathWithDb: string + + before(() => { + // the test config file has no database section; this copy selects Elasticsearch + const withDb = { + ...loadConfigFromFile(TEST_CONFIG_FILE), + dbConfig: { + dbType: 'elasticsearch', + url: ELASTIC_URL, + username: 'elastic', + password: 'changeme' + } + } + configPathWithDb = path.join(os.tmpdir(), `ocean-config-db-${process.pid}.json`) + fs.writeFileSync(configPathWithDb, JSON.stringify(withDb)) + }) + + // returns the config built with the given variables, or the thrown error + async function configWith( + vars: { type?: string; url?: string; username?: string; password?: string }, + configPath?: string + ): Promise<{ config?: OceanNodeConfig; error?: Error }> { + const overrides = buildEnvOverrideConfig( + [ + ENVIRONMENT_VARIABLES.DB_TYPE, + ENVIRONMENT_VARIABLES.DB_URL, + ENVIRONMENT_VARIABLES.DB_USERNAME, + ENVIRONMENT_VARIABLES.DB_PASSWORD, + ...(configPath ? [ENVIRONMENT_VARIABLES.CONFIG_PATH] : []) + ], + [ + vars.type, + vars.url, + vars.username, + vars.password, + ...(configPath ? [configPath] : []) + ] + ) + try { + await setupEnvironment(TEST_ENV_CONFIG_PATH, overrides) + return { config: await getConfiguration(true) } + } catch (error) { + return { error } + } finally { + await tearDownEnvironment(overrides) + } + } + + it('should refuse to start with no DB_TYPE, also when DB_URL is set', async () => { + for (const vars of [ + {}, + { url: ELASTIC_URL, username: 'ocean', password: 'secret' }, + { url: POSTGRES_URL, username: 'ocean', password: 'secret' } + ]) { + const { config: conf, error } = await configWith(vars) + expect(conf).to.be.equal(undefined) + expect(error?.message).to.be.equal('Configuration validation failed') + } + }) + + it('should use SQLite when DB_TYPE is sqlite, and ignore DB_URL', async () => { + const { config: conf, error } = await configWith({ type: 'sqlite', url: ELASTIC_URL }) + expect(error).to.be.equal(undefined) + expect(conf.dbConfig).to.deep.equal({ dbType: 'sqlite' }) + }) + + it('should refuse a database section of the config file that is not an object', async () => { + const file = path.join(os.tmpdir(), `ocean-config-bad-db-${process.pid}.json`) + const config = { ...loadConfigFromFile(TEST_CONFIG_FILE), dbConfig: POSTGRES_URL } + fs.writeFileSync(file, JSON.stringify(config)) + const { config: conf, error } = await configWith({}, file) + fs.rmSync(file, { force: true }) + expect(conf).to.be.equal(undefined) + expect(error?.message).to.be.equal('Configuration validation failed') + }) + + it('should refuse an unknown DB_TYPE', async () => { + const { config: conf, error } = await configWith({ type: 'mongo' }) + expect(conf).to.be.equal(undefined) + expect(error?.message).to.be.equal('Configuration validation failed') + }) + + for (const [type, url] of [ + ['elasticsearch', ELASTIC_URL], + ['postgresql', POSTGRES_URL] + ]) { + it(`should build the ${type} configuration when all three values are set`, async () => { + const { config: conf, error } = await configWith({ + type, + url, + username: 'ocean', + password: 'secret' + }) + expect(error).to.be.equal(undefined) + expect(conf.dbConfig).to.deep.equal({ + dbType: type, + url, + username: 'ocean', + password: 'secret' + }) + }) + + it(`should refuse to start on ${type} when DB_URL is empty`, async () => { + const { config: conf, error } = await configWith({ + type, + url: '', + username: 'ocean', + password: 'secret' + }) + expect(conf).to.be.equal(undefined) + expect(error?.message).to.be.equal('Configuration validation failed') + }) + + it(`should refuse to start on ${type} when DB_PASSWORD is empty`, async () => { + const { config: conf, error } = await configWith({ + type, + url, + username: 'ocean', + password: '' + }) + expect(conf).to.be.equal(undefined) + expect(error?.message).to.be.equal('Configuration validation failed') + }) + } + + it('should keep the config file values for the DB_* variables that are empty', async () => { + // `DB_TYPE=${DB_TYPE}` in a compose file gives an empty value + const { config: conf, error } = await configWith( + { type: '', url: '', username: '', password: '' }, + configPathWithDb + ) + expect(error).to.be.equal(undefined) + expect(conf.dbConfig.dbType).to.be.equal('elasticsearch') + expect(conf.dbConfig.url).to.be.equal(ELASTIC_URL) + }) + + it('should not check the DB_URL scheme: the driver connects to it as given', async () => { + const { config: conf, error } = await configWith({ + type: 'postgresql', + url: 'localhost:5432', + username: 'ocean', + password: 'secret' + }) + expect(error).to.be.equal(undefined) + expect(conf.dbConfig.url).to.be.equal('localhost:5432') + }) + + it('should keep the config file values for the DB_* variables that are not set', async () => { + const { config: conf, error } = await configWith( + { password: 'secret' }, + configPathWithDb + ) + expect(error).to.be.equal(undefined) + expect(conf.dbConfig).to.deep.equal({ + dbType: 'elasticsearch', + url: ELASTIC_URL, + username: 'elastic', + password: 'secret' + }) + }) + + after(() => { + fs.rmSync(configPathWithDb, { force: true }) + delete process.env.CONFIG_PATH + delete process.env.PRIVATE_KEY + }) +}) + describe('Should validate P2P config from environment variables', () => { let config: OceanNodeConfig let envOverrides: OverrideEnvConfig[] @@ -168,23 +346,13 @@ describe('Should validate P2P config from environment variables', () => { before(async () => { envOverrides = buildEnvOverrideConfig( [ - ENVIRONMENT_VARIABLES.DB_TYPE, - ENVIRONMENT_VARIABLES.DB_URL, ENVIRONMENT_VARIABLES.P2P_ipV4BindAddress, ENVIRONMENT_VARIABLES.P2P_ipV4BindTcpPort, ENVIRONMENT_VARIABLES.P2P_ipV6BindAddress, ENVIRONMENT_VARIABLES.P2P_MIN_CONNECTIONS, ENVIRONMENT_VARIABLES.P2P_MAX_CONNECTIONS ], - [ - 'typesense', - 'http://localhost:8108/?apiKey=xyz', - '127.0.0.1', - '9999', - '::2', - '5', - '500' - ] + ['127.0.0.1', '9999', '::2', '5', '500'] ) envOverrides = await setupEnvironment(TEST_ENV_CONFIG_PATH, envOverrides) config = await getConfiguration(true) @@ -229,9 +397,6 @@ describe('JobType enum', () => { }) describe('Should validate SUBSIDY_PROVIDERS configuration', () => { - const DB_ENV_VARS = [ENVIRONMENT_VARIABLES.DB_TYPE, ENVIRONMENT_VARIABLES.DB_URL] - const DB_ENV_VALUES = ['typesense', 'http://localhost:8108/?apiKey=xyz'] - // Mixed/lower-case inputs — the schema must normalize these to EIP-55 checksummed form. const ADDR_A_LOWER = '0x1c7d4b196cb0c7b01d743fbc6116a902379c7238' const ADDR_B_LOWER = '0x0000000000000000000000000000000000000abc' @@ -241,8 +406,8 @@ describe('Should validate SUBSIDY_PROVIDERS configuration', () => { async function configWith( subsidyProviders?: string ): Promise<{ config?: OceanNodeConfig; error?: Error }> { - const envVars = [...DB_ENV_VARS] - const envValues = [...DB_ENV_VALUES] + const envVars: EnvVariable[] = [] + const envValues: string[] = [] if (subsidyProviders !== undefined) { envVars.push(ENVIRONMENT_VARIABLES.SUBSIDY_PROVIDERS) envValues.push(subsidyProviders) @@ -338,9 +503,6 @@ describe('Should validate SUBSIDY_PROVIDERS configuration', () => { }) describe('Should validate SUBSIDY_PROVIDER_ALLOWED_IMAGES configuration', () => { - const DB_ENV_VARS = [ENVIRONMENT_VARIABLES.DB_TYPE, ENVIRONMENT_VARIABLES.DB_URL] - const DB_ENV_VALUES = ['typesense', 'http://localhost:8108/?apiKey=xyz'] - const ADDR_LOWER = '0x1c7d4b196cb0c7b01d743fbc6116a902379c7238' const IMAGE_TAG = 'oceanprotocol/algo:1.0' const IMAGE_DIGEST = 'oceanprotocol/algo@sha256:' + 'a'.repeat(64) @@ -350,8 +512,8 @@ describe('Should validate SUBSIDY_PROVIDER_ALLOWED_IMAGES configuration', () => async function configWith( allowedImages?: string ): Promise<{ config?: OceanNodeConfig; error?: Error }> { - const envVars = [...DB_ENV_VARS] - const envValues = [...DB_ENV_VALUES] + const envVars: EnvVariable[] = [] + const envValues: string[] = [] if (allowedImages !== undefined) { envVars.push(ENVIRONMENT_VARIABLES.SUBSIDY_PROVIDER_ALLOWED_IMAGES) envValues.push(allowedImages) @@ -432,14 +594,11 @@ describe('Should validate SUBSIDY_PROVIDER_ALLOWED_IMAGES configuration', () => }) describe('Should validate SUBSIDY_PROVIDER_FILTER configuration', () => { - const DB_ENV_VARS = [ENVIRONMENT_VARIABLES.DB_TYPE, ENVIRONMENT_VARIABLES.DB_URL] - const DB_ENV_VALUES = ['typesense', 'http://localhost:8108/?apiKey=xyz'] - async function configWith( filter?: string ): Promise<{ config?: OceanNodeConfig; error?: Error }> { - const envVars = [...DB_ENV_VARS] - const envValues = [...DB_ENV_VALUES] + const envVars: EnvVariable[] = [] + const envValues: string[] = [] if (filter !== undefined) { envVars.push(ENVIRONMENT_VARIABLES.SUBSIDY_PROVIDER_FILTER) envValues.push(filter) @@ -486,6 +645,49 @@ describe('Should validate SUBSIDY_PROVIDER_FILTER configuration', () => { }) }) +describe('Should resolve the instance id', () => { + async function instanceIdWith(instanceId?: string): Promise { + const envVars: EnvVariable[] = [] + const envValues: string[] = [] + if (instanceId !== undefined) { + envVars.push(ENVIRONMENT_VARIABLES.INSTANCE_ID) + envValues.push(instanceId) + } + const overrides = buildEnvOverrideConfig(envVars, envValues) + try { + await setupEnvironment(TEST_ENV_CONFIG_PATH, overrides) + const conf = await getConfiguration(true) + const keyManager = new KeyManager(conf) + const node = OceanNode.getInstance( + conf, + null, + null, + null, + null, + keyManager, + null, + true + ) + return node.getInstanceId() + } finally { + await tearDownEnvironment(overrides) + } + } + + it('should use the hostname when INSTANCE_ID is not set', async () => { + expect(await instanceIdWith()).to.be.equal(os.hostname()) + }) + + it('should use INSTANCE_ID when it is set', async () => { + expect(await instanceIdWith('node-a')).to.be.equal('node-a') + }) + + after(() => { + delete process.env.CONFIG_PATH + delete process.env.PRIVATE_KEY + }) +}) + describe('Should validate persistent storage bucket sharing configuration', () => { const PERSISTENT_STORAGE_JSON = JSON.stringify({ enabled: true, @@ -495,16 +697,8 @@ describe('Should validate persistent storage bucket sharing configuration', () = }) async function configWith(allowSharing?: string): Promise { - const envVars = [ - ENVIRONMENT_VARIABLES.DB_TYPE, - ENVIRONMENT_VARIABLES.DB_URL, - ENVIRONMENT_VARIABLES.PERSISTENT_STORAGE - ] - const envValues = [ - 'typesense', - 'http://localhost:8108/?apiKey=xyz', - PERSISTENT_STORAGE_JSON - ] + const envVars = [ENVIRONMENT_VARIABLES.PERSISTENT_STORAGE] + const envValues = [PERSISTENT_STORAGE_JSON] if (allowSharing !== undefined) { envVars.push(ENVIRONMENT_VARIABLES.PERSISTENT_STORAGE_ALLOW_BUCKET_SHARING) envValues.push(allowSharing) @@ -546,17 +740,14 @@ describe('Should validate persistent storage bucket sharing configuration', () = // on the parsed value, otherwise an invalid compute config — e.g. a GPU missing its // mandatory `memoryTotal` (VRAM) — would slip through buildMergedConfig unvalidated. describe('Should validate DOCKER_COMPUTE_ENVIRONMENTS from the environment variable', () => { - const DB_ENV_VARS = [ENVIRONMENT_VARIABLES.DB_TYPE, ENVIRONMENT_VARIABLES.DB_URL] - const DB_ENV_VALUES = ['typesense', 'http://localhost:8108/?apiKey=xyz'] - // Builds the config with DOCKER_COMPUTE_ENVIRONMENTS set to the given JSON string, and // returns the built config or the thrown error. Mirrors the retry-config helper above. async function configWithDockerEnv( dockerEnvJson: string ): Promise<{ config?: OceanNodeConfig; error?: Error }> { const overrides = buildEnvOverrideConfig( - [...DB_ENV_VARS, ENVIRONMENT_VARIABLES.DOCKER_COMPUTE_ENVIRONMENTS], - [...DB_ENV_VALUES, dockerEnvJson] + [ENVIRONMENT_VARIABLES.DOCKER_COMPUTE_ENVIRONMENTS], + [dockerEnvJson] ) try { await setupEnvironment(TEST_ENV_CONFIG_PATH, overrides) diff --git a/src/test/unit/database/databaseInit.test.ts b/src/test/unit/database/databaseInit.test.ts new file mode 100644 index 000000000..909bc04f0 --- /dev/null +++ b/src/test/unit/database/databaseInit.test.ts @@ -0,0 +1,51 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import { Database } from '../../../components/database/index.js' +import { DatabaseFactory } from '../../../components/database/DatabaseFactory.js' +import { + createElasticsearchClientWithRetry, + getOrNull +} from '../../../components/database/ElasticsearchConfigHelper.js' +import { getConfiguration } from '../../../utils/config.js' +import { describeOnElasticsearch } from '../../utils/utils.js' + +describe('Database.init', () => { + afterEach(() => { + sinon.restore() + }) + + it('returns null when a store fails to initialize', async () => { + const config = await getConfiguration() + sinon.stub(DatabaseFactory, 'createStores').rejects(new Error('nonce store down')) + expect(await Database.init(config.dbConfig)).to.be.equal(null) + }) + + it('the factory refuses a database type that has no stores', () => { + expect(() => DatabaseFactory.createStores({ dbType: 'bogus' } as any)).to.throw( + 'Unknown database type: bogus' + ) + }) + + it('exposes the stores that the factory builds', async () => { + const stores: any = { nonce: {}, c2d: {} } + sinon.stub(DatabaseFactory, 'createStores').resolves(stores) + const database = await Database.init({ dbType: 'sqlite' }) + expect(database.nonce).to.be.equal(stores.nonce) + expect(database.c2d).to.be.equal(stores.c2d) + expect(database.getConfig()).to.deep.equal({ dbType: 'sqlite' }) + }) +}) + +describeOnElasticsearch('getOrNull', () => { + it('rejects when the index does not exist', async () => { + const config = await getConfiguration() + const client = await createElasticsearchClientWithRetry(config.dbConfig) + let error: any = null + try { + await getOrNull(client, 'ocean_test_index_that_does_not_exist', 'any-id') + } catch (err) { + error = err + } + expect(error?.meta?.body?.error?.type).to.equal('index_not_found_exception') + }) +}) diff --git a/src/test/unit/database/ddoQuery.test.ts b/src/test/unit/database/ddoQuery.test.ts new file mode 100644 index 000000000..ca3643419 --- /dev/null +++ b/src/test/unit/database/ddoQuery.test.ts @@ -0,0 +1,631 @@ +import { expect } from 'chai' +import { getAddress } from 'ethers' +import type { Client } from '@elastic/elasticsearch' +import { + AbstractDdoDatabase, + QueryError, + TagCount +} from '../../../components/database/BaseDatabase.js' +import { + assetConditions, + pageOf, + parseQueryRequest, + sortOf +} from '../../../components/database/assetQuery.js' +import { SqlDdoDatabase } from '../../../components/database/sql/SqlDdo.js' +import { SqlDocuments } from '../../../components/database/sql/SqlDocuments.js' +import type { SqlClient } from '../../../components/database/sql/SqlClient.js' +import { ElasticsearchDdoDatabase } from '../../../components/database/ElasticSearchDatabase.js' +import { createElasticsearchClientWithRetry } from '../../../components/database/ElasticsearchConfigHelper.js' +import { getConfiguration } from '../../../utils/index.js' +import { SQL_DB_CONFIG, makeSqlClient } from '../../utils/sqlClient.js' +import { describeOnElasticsearch } from '../../utils/utils.js' + +// The queries of the Enterprise Market (OceanProtocolEnterprise/market, src/@utils/aquarius +// and src/@utils/compute), written in the query of the node, give the same DDOs on SQLite, +// PostgreSQL and Elasticsearch, for DDO v4 and v5 documents. + +// a chain of these tests only, so the documents of a shared Elasticsearch do not count +const CHAIN = 7777777 +const address = (byte: string) => getAddress('0x' + byte.repeat(20)) +const OWNER_A = address('a1') +const OWNER_B = address('b2') +const NODE1 = 'https://node1.example.com' +const NODE2 = 'https://node2.example.com' + +interface Stats { + owner?: string + state?: number + purgatory?: boolean + block?: number + orders?: number + priceType?: string + price?: string + symbol?: string + tokenName?: string + datatoken?: string +} + +const indexed = (s: Stats) => ({ + nft: { state: s.state ?? 0, owner: s.owner }, + purgatory: { state: s.purgatory ?? false }, + event: { block: s.block }, + stats: [ + { + datatokenAddress: s.datatoken, + orders: s.orders, + name: s.tokenName, + symbol: s.symbol, + prices: [{ type: s.priceType, price: s.price }] + } + ] +}) + +const v5 = ( + id: string, + subject: Record, + stats?: Stats +): Record => ({ + '@context': ['https://www.w3.org/ns/credentials/v2'], + id, + version: '5.0.0', + credentialSubject: { id, chainId: CHAIN, ...subject }, + ...(stats && { indexedMetadata: indexed(stats) }) +}) + +const A = v5( + 'did:op:query-a', + { + nftAddress: address('e1'), + dataspace: 'ds-1', + metadata: { + type: 'dataset', + name: 'Ocean Weather Data', + description: { value: 'Hourly temperature of the North Sea' }, + author: 'OPF', + tags: ['weather', 'climate'], + created: '2026-01-10T00:00:00Z' + }, + services: [ + { type: 'access', datatokenAddress: address('d1'), serviceEndpoint: NODE1 }, + { + type: 'compute', + datatokenAddress: address('d2'), + serviceEndpoint: NODE1, + compute: { + publisherTrustedAlgorithms: [{ did: 'did:op:query-b', serviceId: 'algo-svc' }], + publisherTrustedAlgorithmPublishers: [OWNER_B] + } + } + ] + }, + { + owner: OWNER_A, + block: 100, + orders: 5, + priceType: 'fixedrate', + price: '8', + symbol: 'WTH', + tokenName: 'Weather Token', + datatoken: address('d1') + } +) +const B = v5( + 'did:op:query-b', + { + nftAddress: address('e2'), + dataspace: 'ds-1', + metadata: { + type: 'algorithm', + name: 'Image Classifier', + description: { value: 'Classifies images' }, + author: 'OPF', + tags: ['ml'], + created: '2026-02-10T00:00:00Z' + }, + services: [ + { type: 'access', datatokenAddress: address('d3'), serviceEndpoint: NODE2 } + ] + }, + { + owner: OWNER_B, + block: 200, + orders: 12, + priceType: 'dispenser', + price: '0', + symbol: 'IMG', + tokenName: 'Image Token' + } +) +const C = v5( + 'did:op:query-c', + { + nftAddress: address('e3'), + dataspace: 'ds-2', + metadata: { + type: 'dataset', + name: 'Sea level archive', + description: { value: 'Tide gauges' }, + author: 'OPF', + tags: ['climate'], + created: '2026-03-10T00:00:00Z' + }, + services: [ + { + type: 'compute', + datatokenAddress: address('d4'), + serviceEndpoint: NODE2, + compute: { + publisherTrustedAlgorithms: [{ did: '*', serviceId: '*' }], + publisherTrustedAlgorithmPublishers: ['*'] + } + } + ] + }, + { + owner: OWNER_A, + purgatory: true, + block: 300, + orders: 0, + priceType: 'fixedrate', + price: '3', + symbol: 'SEA' + } +) +const D = v5( + 'did:op:query-d', + { + nftAddress: address('e4'), + dataspace: 'ds-1', + metadata: { + type: 'dataset', + name: 'Unlisted weather set', + description: { value: 'Unlisted' }, + author: 'OPF', + tags: ['weather'], + created: '2026-04-10T00:00:00Z' + }, + services: [ + { type: 'access', datatokenAddress: address('d5'), serviceEndpoint: NODE1 } + ] + }, + { + owner: OWNER_B, + state: 5, + block: 400, + orders: 1, + priceType: 'pool', + price: '7', + symbol: 'REV' + } +) +const E = { + '@context': ['https://w3id.org/did/v1'], + id: 'did:op:query-e', + version: '4.1.0', + chainId: CHAIN, + nftAddress: address('e5'), + metadata: { + type: 'dataset', + name: 'Weather stations', + description: 'Daily weather station readings', + author: 'Met', + tags: ['weather'], + created: '2025-12-01T00:00:00Z' + }, + services: [{ type: 'access', datatokenAddress: address('d6'), serviceEndpoint: NODE1 }], + indexedMetadata: indexed({ + owner: OWNER_A, + block: 50, + orders: 2, + priceType: 'fixedrate', + price: '5', + symbol: 'STN' + }) +} +// not indexed yet: no indexedMetadata +const F = v5('did:op:query-f', { + nftAddress: address('e6'), + dataspace: 'ds-1', + metadata: { + type: 'dataset', + name: 'Unindexed weather', + description: { value: 'Pending' }, + author: 'OPF', + tags: [] as string[], + created: '2026-05-01T00:00:00Z' + }, + services: [{ type: 'access', datatokenAddress: address('d7'), serviceEndpoint: NODE1 }] +}) + +const DOCUMENTS: Record[] = [A, B, C, D, E, F] + +interface Case { + name: string + body: Record + // the ids in this order; `anyOrder` compares them as a set + ids?: string[] + anyOrder?: boolean + total?: number + tags?: TagCount[] +} + +const onChain = (filter: Record = {}) => ({ chainIds: [CHAIN], ...filter }) + +const CASES: Case[] = [ + { + name: 'search box: the assets with the word, newest first', + body: { + filter: onChain({ text: 'weather' }), + sort: { by: 'created', order: 'desc' } + }, + ids: [A.id, E.id, F.id] + }, + { + name: 'search box: every word must be in the asset', + body: { filter: onChain({ text: 'Weather NORTH' }) }, + ids: [A.id] + }, + { + name: 'search box: the start of a word', + body: { filter: onChain({ text: 'classif' }) }, + ids: [B.id] + }, + { + name: 'search box with relevance sort', + body: { filter: onChain({ text: 'weather' }), sort: { by: 'relevance' } }, + ids: [A.id, E.id, F.id], + anyOrder: true + }, + { + name: 'relevance with no text: newest first', + body: { filter: onChain(), sort: { by: 'relevance' } }, + ids: [B.id, A.id, E.id, F.id] + }, + { + name: 'search box: a pasted DID', + body: { filter: onChain({ text: ` ${B.id} ` }) }, + ids: [B.id] + }, + { + name: 'search box: a pasted owner address, in lower case', + body: { filter: onChain({ text: OWNER_B.toLowerCase() }) }, + ids: [B.id] + }, + { + name: 'search box: a pasted datatoken address', + body: { filter: onChain({ text: address('d1') }) }, + ids: [A.id] + }, + { + name: 'DDO v5 only', + body: { filter: onChain({ versions: ['5.0.0'] }) }, + ids: [B.id, A.id, F.id] + }, + { + name: 'search filters: type, service, tags, price, time, node and dataspace', + body: { + filter: onChain({ + types: ['dataset'], + serviceTypes: ['access'], + tags: ['weather'], + priceTypes: ['fixedrate'], + createdAfter: '2026-01-01T00:00:00.000Z', + serviceEndpoints: [NODE1], + dataspace: 'ds-1' + }) + }, + ids: [A.id] + }, + { + name: 'the default hides unlisted (state 5) and purgatory assets', + body: { filter: onChain() }, + ids: [B.id, A.id, E.id, F.id] + }, + { + name: 'hide pool prices, with an explicit list of states', + body: { + filter: onChain({ + excludePriceTypes: ['pool'], + nftStates: [0, 5], + includeUnlisted: true + }) + }, + ids: [B.id, A.id, E.id] + }, + { + name: 'asset state filter: unlisted stays hidden unless included', + body: { filter: onChain({ nftStates: [5] }) }, + ids: [] + }, + { + name: 'asset state filter with unlisted included', + body: { filter: onChain({ nftStates: [5], includeUnlisted: true }) }, + ids: [D.id] + }, + { + name: 'published assets of an owner, with purgatory, owner in lower case', + body: { + filter: onChain({ owners: [OWNER_A.toLowerCase()], includePurgatory: true }) + }, + ids: [C.id, A.id, E.id] + }, + { + name: 'bookmarks and most viewed: assets from a DID list', + body: { filter: onChain({ ids: [A.id, B.id, 'did:op:none'] }) }, + ids: [B.id, A.id] + }, + { + name: 'related assets: same tags, without this asset', + body: { + filter: onChain({ tags: ['weather', 'climate'], excludeIds: [A.id] }), + size: 4 + }, + ids: [E.id] + }, + { + name: 'related assets: same owner, without this NFT, most orders first', + body: { + filter: onChain({ + owners: [OWNER_A], + excludeNftAddresses: [A.credentialSubject.nftAddress.toLowerCase()] + }), + sort: { by: 'orders', order: 'desc' } + }, + ids: [E.id] + }, + { + name: 'algorithm allowlist: one of the ids or of the owners', + body: { + filter: onChain({ + types: ['algorithm'], + anyOf: [{ ids: ['did:op:none'], owners: [OWNER_B] }] + }), + size: 3000 + }, + ids: [B.id] + }, + { + name: 'whitelist and allowlist: one value of each group', + body: { + filter: onChain({ + anyOf: [ + { owners: [OWNER_B], datatokenAddresses: [address('d1').toLowerCase()] }, + { ids: [A.id, 'did:op:none'] } + ] + }) + }, + ids: [A.id] + }, + { + name: 'datasets that trust an algorithm (DID and service)', + body: { + filter: onChain({ trustedAlgorithm: { did: B.id, serviceId: 'algo-svc' } }) + }, + ids: [A.id] + }, + { + name: 'datasets that trust every algorithm', + body: { + filter: onChain({ + trustedAlgorithm: { did: '*', serviceId: '*' }, + includePurgatory: true + }) + }, + ids: [C.id] + }, + { + name: 'datasets that trust every algorithm publisher', + body: { filter: onChain({ trustedAlgorithmPublisher: '*', includePurgatory: true }) }, + ids: [C.id] + }, + { + name: 'compute datasets of a datatoken, address in lower case', + body: { + filter: onChain({ + datatokenAddresses: [ + A.credentialSubject.services[1].datatokenAddress.toLowerCase() + ], + serviceTypes: ['compute'], + types: ['dataset'], + includePurgatory: true + }) + }, + ids: [A.id] + }, + { + name: 'downloads: indexed access assets of datatokens', + body: { + filter: onChain({ + datatokenAddresses: [ + E.services[0].datatokenAddress, + F.credentialSubject.services[0].datatokenAddress + ], + serviceTypes: ['access'], + indexedOnly: true, + includePurgatory: true + }) + }, + ids: [E.id] + }, + { + name: 'sort by orders, the most first; without the field last', + body: { filter: onChain(), sort: { by: 'orders', order: 'desc' } }, + ids: [B.id, A.id, E.id, F.id] + }, + { + name: 'sort by price, the lowest first', + body: { filter: onChain(), sort: { by: 'price', order: 'asc' } }, + ids: [B.id, E.id, A.id, F.id] + }, + { + name: 'sort by token symbol', + body: { filter: onChain(), sort: { by: 'tokenSymbol', order: 'asc' } }, + ids: [B.id, E.id, A.id, F.id] + }, + { + name: 'sort by created, the oldest first', + body: { filter: onChain(), sort: { by: 'created', order: 'asc' } }, + ids: [E.id, A.id, B.id, F.id] + }, + { + name: 'one page, and the total of all matches', + body: { filter: onChain(), page: 2, size: 2 }, + ids: [E.id, F.id], + total: 4 + }, + { + name: 'tags of the assets that match', + body: { type: 'tags', filter: onChain() }, + tags: [ + { tag: 'weather', count: 2 }, + { tag: 'climate', count: 1 }, + { tag: 'ml', count: 1 } + ] + } +] + +async function run(store: AbstractDdoDatabase, body: Record) { + const request = parseQueryRequest(body) + if (request.type === 'tags') { + return { tags: await store.tags(assetConditions(request.filter), request.size) } + } + if (request.type !== 'assets') throw new Error('not a DDO query') + return await store.find( + assetConditions(request.filter), + sortOf(request.sort), + pageOf(request) + ) +} + +function checkCases(getStore: () => AbstractDdoDatabase): void { + for (const c of CASES) { + it(c.name, async () => { + const answer: any = await run(getStore(), c.body) + if (c.tags) { + expect(answer.tags).to.deep.equal(c.tags) + return + } + const ids = answer.results.map((ddo: any) => ddo.id) + if (c.anyOrder) expect([...ids].sort()).to.deep.equal([...c.ids].sort()) + else expect(ids).to.deep.equal(c.ids) + if (c.total !== undefined) expect(answer.totalResults).to.equal(c.total) + }) + } +} + +describe('Query of the node: the Enterprise Market searches on SQL', () => { + let db: SqlClient + let store: SqlDdoDatabase + + before(async () => { + db = await makeSqlClient() + store = new SqlDdoDatabase(SQL_DB_CONFIG, db) + await store.init() + // stored as they are: the SHACL validation of create() is not under test here + const documents = new SqlDocuments(db, 'ddo') + for (const doc of DOCUMENTS) await documents.put(doc.id, doc) + }) + after(() => db.close()) + + checkCases(() => store) + + it('deletes the DDOs of a chain, v4 and v5', async () => { + expect(await store.deleteAllAssetsFromChain(CHAIN)).to.equal(DOCUMENTS.length) + }) +}) + +describeOnElasticsearch( + 'Query of the node: the Enterprise Market searches on Elasticsearch', + () => { + let store: ElasticsearchDdoDatabase + let client: Client + + before(async function () { + this.timeout(60000) + const config = (await getConfiguration()).dbConfig + store = await ElasticsearchDdoDatabase.create(config) + client = await createElasticsearchClientWithRetry(config) + await store.deleteAllAssetsFromChain(CHAIN) + // stored as they are, in the index of their version + for (const doc of DOCUMENTS) { + await client.index({ + index: doc.version === '5.0.0' ? 'op_ddo_v5.0.0' : 'op_ddo_v4.1.0', + id: doc.id, + document: doc, + refresh: 'wait_for' + }) + } + }) + after(async () => { + await store.deleteAllAssetsFromChain(CHAIN) + }) + + checkCases(() => store) + } +) + +describe('Query of the node: the request', () => { + const invalid = (body: unknown) => { + try { + parseQueryRequest(body) + } catch (error) { + expect(error).to.be.instanceOf(QueryError) + return error.message + } + throw new Error('no error') + } + + it('fills the defaults: assets, page 1, size 1000, newest first', () => { + expect(parseQueryRequest({})).to.deep.equal({ + type: 'assets', + filter: {}, + page: 1, + size: 1000 + }) + expect(sortOf(undefined)).to.deep.equal([ + { paths: ['indexedMetadata.event.block'], order: 'desc' } + ]) + }) + + it('refuses an Elasticsearch body, and names the new format', () => { + expect(invalid({ query: { match_all: {} }, from: 1, size: 10 })).to.contain( + '{ type, filter, sort, page, size }' + ) + expect(invalid({ index: 'order' })).to.contain('Elasticsearch') + }) + + it('refuses an unknown filter, a wrong value and an unknown type', () => { + expect(invalid({ filter: { color: 'red' } })).to.contain('color') + expect(invalid({ filter: { types: ['movie'] } })).to.contain('filter.types') + expect(invalid({ filter: { createdAfter: 'yesterday' } })).to.contain('ISO date') + expect(invalid({ type: 'logs' })).to.contain('type') + }) + + it('refuses a page past the first 10000 documents', () => { + expect(invalid({ page: 101, size: 100 })).to.contain('10000') + expect(parseQueryRequest({ page: 100, size: 100 })).to.include({ page: 100 }) + }) + + it('an orders query needs a consumer or a payer', () => { + expect(invalid({ type: 'orders', filter: {} })).to.contain('filter') + expect(parseQueryRequest({ type: 'orders', filter: { payer: '0x1' } })).to.deep.equal( + { + type: 'orders', + filter: { payer: '0x1' }, + page: 1, + size: 1000 + } + ) + }) + + it('maps each filter to its condition; unset filters add none', () => { + expect( + assetConditions({ includePurgatory: true, includeUnlisted: true, nftStates: [0] }) + ).to.deep.equal([{ paths: ['indexedMetadata.nft.state'], anyOf: [0] }]) + expect(assetConditions({})).to.deep.equal([ + { paths: ['indexedMetadata.nft.state'], noneOf: [5] }, + { paths: ['indexedMetadata.purgatory.state'], noneOf: [true] } + ]) + }) +}) diff --git a/src/test/unit/database/getJobsFilters.test.ts b/src/test/unit/database/getJobsFilters.test.ts index 6d2ac8639..ceda1a603 100644 --- a/src/test/unit/database/getJobsFilters.test.ts +++ b/src/test/unit/database/getJobsFilters.test.ts @@ -1,6 +1,9 @@ import { assert, expect } from 'chai' +import { Client } from '@elastic/elasticsearch' import { C2DDatabase } from '../../../components/database/C2DDatabase.js' -import { typesenseSchemas } from '../../../components/database/TypesenseSchemas.js' +import { Database } from '../../../components/database/index.js' +import { createElasticsearchClientWithRetry } from '../../../components/database/ElasticsearchConfigHelper.js' +import { stateSchemas } from '../../../components/database/ElasticSchemas.js' import { getConfiguration } from '../../../utils/config.js' import { C2DStatusNumber, @@ -14,7 +17,8 @@ import { OverrideEnvConfig, setupEnvironment, tearDownEnvironment, - TEST_ENV_CONFIG_FILE + TEST_ENV_CONFIG_FILE, + describeOnElasticsearch } from '../../utils/utils.js' import { ENVIRONMENT_VARIABLES, PROTOCOL_COMMANDS } from '../../../utils/constants.js' import { OceanNodeConfig } from '../../../@types/OceanNode.js' @@ -39,11 +43,6 @@ const OLD_SEC = BOUND_SEC - 7200 const NEW_FINISHED = '1785760660.961' const NINE_DIGIT = '999999999.5' -// The C2D SQLite file survives between test runs, so every environment/jobId below is -// namespaced per run: each test then sees only the rows it seeded. -const RUN = String(Date.now()) -const envName = (base: string) => `${base}-${RUN}` - const algorithm: ComputeAlgorithm = { documentId: 'did:op:1', serviceId: '0xabc' } const dataset: ComputeAsset = { documentId: 'did:op:2', serviceId: '0xdef' } @@ -82,10 +81,11 @@ function baseJob(overrides: Partial): DBComputeJob { } as unknown as DBComputeJob } -describe('getJobs filters', () => { +describeOnElasticsearch('getJobs filters', () => { let envOverrides: OverrideEnvConfig[] let config: OceanNodeConfig let db: C2DDatabase = null + let client: Client before(async () => { envOverrides = buildEnvOverrideConfig( @@ -96,7 +96,22 @@ describe('getJobs filters', () => { ) envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) config = await getConfiguration(true) - db = await new C2DDatabase(config.dbConfig, typesenseSchemas.c2dSchemas) + db = (await Database.init(config.dbConfig)).c2d + client = await createElasticsearchClientWithRetry(config.dbConfig) + }) + + // every write uses refresh: 'wait_for', so no index refresh is needed first + beforeEach(async () => { + await client.deleteByQuery({ + index: [ + stateSchemas.c2dJobs.index, + stateSchemas.serviceJobs.index, + stateSchemas.dockerImages.index + ], + query: { match_all: {} }, + refresh: true, + conflicts: 'proceed' + }) }) after(async () => { @@ -106,34 +121,35 @@ describe('getJobs filters', () => { // Seeds finished jobs into a run-private environment, with dateCreated == dateFinished so // one fixture serves both the fromTimestamp filter (dateFinished) and the ordering // assertion (dateCreated). - async function seedFinished(envId: string, finishedSecs: string[]): Promise { - const ids: string[] = [] - for (const [i, finished] of finishedSecs.entries()) { - const jobId = await db.newJob( - baseJob({ - jobId: `${envId}-${i}`, - environment: envId, - dateCreated: finished - }) - ) - // dateFinished is only written by updateJob - await db.updateJob( - baseJob({ - jobId, - environment: envId, - dateCreated: finished, - dateFinished: finished - }) - ) - ids.push(jobId) - } - return ids + // Jobs are seeded in parallel: each write waits for the next index refresh. + function seedFinished(envId: string, finishedSecs: string[]): Promise { + return Promise.all( + finishedSecs.map(async (finished, i) => { + const jobId = await db.newJob( + baseJob({ + jobId: `${envId}-${i}`, + environment: envId, + dateCreated: finished + }) + ) + // dateFinished is only written by updateJob + await db.updateJob( + baseJob({ + jobId, + environment: envId, + dateCreated: finished, + dateFinished: finished + }) + ) + return jobId + }) + ) } describe('fromTimestamp is compared numerically', () => { - const envId = envName('env-fromTimestamp') + const envId = 'env-fromTimestamp' - before(async () => { + beforeEach(async () => { await seedFinished(envId, [String(OLD_SEC), NINE_DIGIT, NEW_FINISHED]) }) @@ -172,7 +188,7 @@ describe('getJobs filters', () => { }) it('includes a row sitting exactly on the bound', async () => { - const envId = envName('env-boundary') + const envId = 'env-boundary' // the filter is >=, so an exact match must be returned await seedFinished(envId, [String(BOUND_SEC)]) const jobs = await db.getJobs([envId], BOUND_SEC) @@ -180,7 +196,7 @@ describe('getJobs filters', () => { }) it('orders by numeric dateCreated, not lexicographically', async () => { - const envId = envName('env-ordering') + const envId = 'env-ordering' // 9-digit vs 10-digit: as text "999999999.5" sorts above "1785760660.961", so pre-fix // the OLDER job came back first under ORDER BY dateCreated DESC. const tenDigits = NEW_FINISHED @@ -194,9 +210,9 @@ describe('getJobs filters', () => { }) describe('status filter', () => { - const envId = envName('env-status') + const envId = 'env-status' - before(async () => { + beforeEach(async () => { // JobStarted is 0 — the value a truthiness guard drops await db.newJob( baseJob({ @@ -245,6 +261,90 @@ describe('getJobs filters', () => { assert(jobs.length === 2, `expected 2 jobs, got ${jobs.length}`) }) }) + + describe('getJob matching', () => { + beforeEach(async () => { + await db.newJob(baseJob({ jobId: 'job-match', agreementId: '0xabc123' })) + }) + + it('finds a job by agreementId given without 0x', async () => { + const jobs = await db.getJob(undefined, 'abc123') + expect(jobs.map((j) => j.jobId)).to.deep.equal(['job-match']) + }) + + it('finds a checksummed owner by its lower-cased address', async () => { + const owner = baseJob({}).owner.toLowerCase() + const jobs = await db.getJob(undefined, undefined, owner) + expect(jobs.map((j) => j.jobId)).to.deep.equal(['job-match']) + expect(jobs[0].owner).to.equal(baseJob({}).owner) + }) + + it('applies agreementId and owner to a read by jobId', async () => { + const owner = baseJob({}).owner.toLowerCase() + const found = await db.getJob('job-match', 'abc123', owner) + expect(found.map((j) => j.jobId)).to.deep.equal(['job-match']) + expect(await db.getJob('job-match', '0xother')).to.deep.equal([]) + expect(await db.getJob('job-match', undefined, '0xsomeoneelse')).to.deep.equal([]) + expect(await db.getJob('missing-job')).to.deep.equal([]) + }) + + it('fills jobIdHash and maxJobDuration on read', async () => { + const [job] = await db.getJob('job-match') + expect(job.jobIdHash).to.be.a('string') + expect(job.maxJobDuration).to.equal(3600) + }) + }) + + it('updateJob sets dateFinished: running excludes it, finished lists newest first', async () => { + const envId = 'env-finish' + await db.newJob(baseJob({ jobId: 'running', environment: envId, dateCreated: '100' })) + const [older, newer] = await seedFinished(envId, ['200.5', '300.25']) + + const running = await db.getRunningJobs() + expect(running.map((j) => j.jobId)).to.deep.equal(['running']) + expect(running[0].dateFinished).to.equal(null) + + const finished = await db.getFinishedJobs([envId]) + expect(finished.map((j) => j.jobId)).to.deep.equal([newer, older]) + expect(Number(finished[0].dateFinished)).to.equal(300.25) + expect(await db.getFinishedJobs(['other-env'])).to.deep.equal([]) + }) + + it('getJobsByStatus filters on environments and statuses', async () => { + const envId = 'env-by-status' + await db.newJob( + baseJob({ + jobId: 'started', + environment: envId, + status: C2DStatusNumber.JobStarted, + dateCreated: '10' + }) + ) + await db.newJob(baseJob({ jobId: 'finished', environment: envId, dateCreated: '20' })) + await db.newJob(baseJob({ jobId: 'elsewhere', environment: 'other-env' })) + + const started = await db.getJobsByStatus([envId], [C2DStatusNumber.JobStarted]) + expect(started.map((j) => j.jobId)).to.deep.equal(['started']) + const all = await db.getJobsByStatus([envId], []) + expect(all.map((j) => j.jobId)).to.deep.equal(['finished', 'started']) + }) + + it('deleteJob returns true once, then false', async () => { + const jobId = await db.newJob(baseJob({ jobId: 'to-delete' })) + expect(await db.deleteJob(jobId)).to.equal(true) + expect(await db.deleteJob(jobId)).to.equal(false) + expect(await db.getJob(jobId)).to.deep.equal([]) + }) + + it('tracks docker image usage', async () => { + const image = 'quay.io/test/image:latest' + await db.updateImage(image) + expect(await db.getOldImages(7)).to.deep.equal([]) + // a negative retention puts the limit in the future + expect(await db.getOldImages(-1)).to.deep.equal([image]) + await db.deleteImage(image) + expect(await db.getOldImages(-1)).to.deep.equal([]) + }) }) describe('GetJobsHandler.validate fromTimestamp', () => { diff --git a/src/test/unit/database/indexStores.test.ts b/src/test/unit/database/indexStores.test.ts new file mode 100644 index 000000000..815e930bc --- /dev/null +++ b/src/test/unit/database/indexStores.test.ts @@ -0,0 +1,433 @@ +import { expect } from 'chai' +import { createHash } from 'node:crypto' +import { getAddress } from 'ethers' +import type { EscrowEvent } from '../../../@types/Escrow.js' +import type { SqlClient } from '../../../components/database/sql/SqlClient.js' +import { SQL_DB_CONFIG, makeSqlClient } from '../../utils/sqlClient.js' +import { + SqlIndexStores, + buildSqlIndexStores +} from '../../../components/database/sql/SqlStores.js' +import { DDOExample, ddov5 as ddov5Example } from '../../data/ddo.js' +import type { Condition, SortSpec } from '../../../components/database/BaseDatabase.js' + +// The contract of the index stores on a SQL database. Each test gets a new, empty database. +const CONTRACT = '0xAbC0000000000000000000000000000000000001' +const WALLET = '0xDeF0000000000000000000000000000000000002' +// The two example DDOs have the same id. A valid id is the hash of the NFT address and the +// chain, so the second DDO goes to another chain. +const V5_CHAIN = 8996 +const ddov5 = { + ...ddov5Example, + chainId: V5_CHAIN, + id: + 'did:op:' + + createHash('sha256') + .update(getAddress(ddov5Example.nftAddress) + V5_CHAIN) + .digest('hex') +} + +describe('SQL index stores', () => { + let db: SqlClient + let stores: SqlIndexStores + + beforeEach(async () => { + db = await makeSqlClient() + stores = await buildSqlIndexStores(SQL_DB_CONFIG, db) + }) + afterEach(() => db.close()) + + it('two nodes that start at the same time can make the tables', async () => { + const other = await makeSqlClient() + const [built] = await Promise.all( + Array.from({ length: 6 }, () => buildSqlIndexStores(SQL_DB_CONFIG, other)) + ) + await built.indexer.update(8996, 20) + expect(await built.indexer.retrieve(8996)).to.deep.equal({ lastIndexedBlock: 20 }) + await other.close() + }) + + describe('indexer', () => { + it('keeps the last indexed block of a chain', async () => { + expect(await stores.indexer.retrieve(8996)).to.equal(null) + expect(await stores.indexer.create(8996, 10)).to.deep.equal({ + id: '8996', + lastIndexedBlock: 10 + }) + await stores.indexer.update(8996, 20) + expect(await stores.indexer.retrieve(8996)).to.deep.equal({ lastIndexedBlock: 20 }) + await stores.indexer.update(137, 5) + expect(await stores.indexer.retrieve(137)).to.deep.equal({ lastIndexedBlock: 5 }) + }) + + it('deletes the entry of a chain; null when there is none', async () => { + await stores.indexer.create(8996, 10) + expect(await stores.indexer.delete(8996)).to.deep.equal({ id: '8996' }) + expect(await stores.indexer.delete(8996)).to.equal(null) + }) + }) + + describe('ddo state', () => { + it('creates a state and finds it by did, nft and txId', async () => { + const created = await stores.ddoState.create( + 8996, + 'did:op:1', + '0xNft', + '0xTx', + true + ) + expect(created).to.deep.equal({ + id: 'did:op:1', + chainId: 8996, + nft: '0xNft', + txId: '0xTx', + valid: true, + error: ' ' + }) + expect((await stores.ddoState.retrieve('did:op:1')).did).to.equal('did:op:1') + for (const key of [{ did: 'did:op:1' }, { nft: '0xNft' }, { txId: '0xTx' }]) { + const found = await stores.ddoState.find(key) + expect(found.map((state: any) => state.id)).to.deep.equal(['did:op:1']) + } + expect(await stores.ddoState.find({ did: 'none' })).to.deep.equal([]) + }) + + it('an update replaces the state, and creates it when absent', async () => { + await stores.ddoState.update(8996, 'did:op:1', '0xNft', '0xTx', false, 'bad ddo') + const state = await stores.ddoState.retrieve('did:op:1') + expect(state.valid).to.equal(false) + expect(state.error).to.equal('bad ddo') + expect(await stores.ddoState.delete('did:op:1')).to.deep.equal({ id: 'did:op:1' }) + expect(await stores.ddoState.retrieve('did:op:1')).to.equal(null) + }) + }) + + describe('order', () => { + const create = (orderId: string, consumer: string, timestamp: number) => + stores.order.create( + orderId, + 'startOrder', + timestamp, + consumer, + '0xPayer', + '0xDatatoken', + '0xNft', + 'did:op:1' + ) + + it('creates an order and reads it back with its id', async () => { + const order = await create('0xorder1', '0xConsumer', 100) + expect(order.orderId).to.equal('0xorder1') + expect(await stores.order.retrieve('0xorder1')).to.deep.equal({ + orderId: '0xorder1', + type: 'startOrder', + timestamp: 100, + consumer: '0xConsumer', + payer: '0xPayer', + datatokenAddress: '0xDatatoken', + nftAddress: '0xNft', + did: 'did:op:1', + id: '0xorder1' + }) + expect(await stores.order.retrieve('none')).to.equal(null) + }) + + it('an update changes the given fields only', async () => { + await create('0xorder1', '0xConsumer', 100) + await stores.order.update('0xorder1', 'reuseOrder', 200, '0xConsumer', '0xPayer2') + const order = await stores.order.retrieve('0xorder1') + expect(order.type).to.equal('reuseOrder') + expect(order.payer).to.equal('0xPayer2') + expect(order.datatokenAddress).to.equal('0xDatatoken') + expect(order.did).to.equal('did:op:1') + }) + + it('finds the orders of a consumer or a payer, the newest first', async () => { + await create('0xorder1', '0xConsumer', 100) + await create('0xorder2', '0xOther', 200) + await create('0xorder3', '0xConsumer', 300) + const all = { offset: 0, limit: 10 } + const found = await stores.order.find({ consumer: '0xconsumer' }, all) + expect(found.results.map((order) => order.orderId)).to.deep.equal([ + '0xorder3', + '0xorder1' + ]) + expect(found.totalResults).to.equal(2) + const page = await stores.order.find({ payer: '0xPayer' }, { offset: 1, limit: 1 }) + expect(page.results.map((order) => order.orderId)).to.deep.equal(['0xorder2']) + expect(page.totalResults).to.equal(3) + }) + + it('deletes an order; null when there is none', async () => { + await create('0xorder1', '0xConsumer', 100) + expect(await stores.order.delete('0xorder1')).to.deep.equal({ id: '0xorder1' }) + expect(await stores.order.delete('0xorder1')).to.equal(null) + }) + }) + + describe('escrow', () => { + const event = (id: string, overrides: Partial = {}): EscrowEvent => ({ + id, + eventType: 'Lock', + chainId: 8996, + contract: '0xEscrow', + block: 100, + txHash: `0xtx-${id}`, + payer: '0xPayer', + amount: '1000000000000000000000', + ...overrides + }) + const search = async (filters: Record, offset?: number, size?: number) => + (await stores.escrow.search(filters, offset, size)).map((e) => e.id).sort() + + beforeEach(async () => { + await stores.escrow.create(event('a')) + await stores.escrow.create(event('b', { block: 200, payer: '0xOther' })) + await stores.escrow.create(event('c', { block: 1000, eventType: 'Claimed' })) + }) + + it('reads an event back, and deletes it', async () => { + expect(await stores.escrow.retrieve('a')).to.deep.equal(event('a')) + expect(await stores.escrow.delete('a')).to.deep.equal({ id: 'a' }) + expect(await stores.escrow.retrieve('a')).to.equal(null) + expect(await stores.escrow.delete('a')).to.equal(null) + }) + + it('filters by equality, and by several values for one field', async () => { + expect(await search({})).to.deep.equal(['a', 'b', 'c']) + expect(await search({ payer: '0xPayer' })).to.deep.equal(['a', 'c']) + expect(await search({ payer: '0xPayer', eventType: 'Claimed' })).to.deep.equal([ + 'c' + ]) + expect(await search({ payer: undefined, eventType: '' })).to.deep.equal([ + 'a', + 'b', + 'c' + ]) + }) + + it('filters a block range as a number', async () => { + // the bounds come as text; as text, '1000' is below '200' + expect(await search({ block: { min: '150', max: '1000' } })).to.deep.equal([ + 'b', + 'c' + ]) + expect(await search({ block: { gt: '100', lt: '1000' } })).to.deep.equal(['b']) + expect(await search({ chainId: [8996, { min: '1' }] })).to.deep.equal([ + 'a', + 'b', + 'c' + ]) + }) + + it('gives one page of the result', async () => { + expect(await search({}, 0, 2)).to.have.length(2) + expect(await search({}, 2, 2)).to.have.length(1) + }) + }) + + describe('access list', () => { + const user = (tokenId: number, wallet = WALLET) => ({ + wallet, + tokenId, + block: 10, + txId: '0xtx' + }) + + it('creates a list and reads it back under the lower-case contract address', async () => { + const created = await stores.accessList.create( + 8996, + CONTRACT, + true, + 5, + '0xdeploy', + 'L' + ) + expect(created).to.deep.equal({ id: `8996-${CONTRACT.toLowerCase()}` }) + expect(await stores.accessList.retrieve(8996, CONTRACT)).to.deep.equal({ + id: `8996-${CONTRACT.toLowerCase()}`, + chainId: 8996, + contractAddress: CONTRACT.toLowerCase(), + users: [], + transferable: true, + deploymentBlock: 5, + deploymentTxId: '0xdeploy', + name: 'L' + }) + expect(await stores.accessList.retrieve(8996, WALLET)).to.equal(null) + }) + + it('adds a token id one time, with the wallet in lower case', async () => { + await stores.accessList.create(8996, CONTRACT, true, 5, '0xdeploy') + await stores.accessList.addUser(8996, CONTRACT, user(1)) + await stores.accessList.addUser(8996, CONTRACT, user(1)) + await stores.accessList.addUser(8996, CONTRACT, user(2)) + const list = await stores.accessList.retrieve(8996, CONTRACT) + expect(list.users).to.deep.equal([ + user(1, WALLET.toLowerCase()), + user(2, WALLET.toLowerCase()) + ]) + }) + + it('a user that comes before the deployment makes the list, and the deployment keeps the user', async () => { + await stores.accessList.addUser(8996, CONTRACT, user(1)) + await stores.accessList.create(8996, CONTRACT, true, 5, '0xdeploy', 'L', 'S') + const list = await stores.accessList.retrieve(8996, CONTRACT) + expect(list.users).to.have.length(1) + expect(list.transferable).to.equal(true) + expect(list.symbol).to.equal('S') + }) + + it('keeps all the users that make a new list at the same time', async () => { + const tokenIds = Array.from({ length: 20 }, (_, tokenId) => tokenId) + await Promise.all( + tokenIds.map((tokenId) => + stores.accessList.addUser(8996, CONTRACT, user(tokenId)) + ) + ) + const list = await stores.accessList.retrieve(8996, CONTRACT) + expect(list.users).to.have.length(20) + }) + + it('removes a user by token id; null for a list that does not exist', async () => { + await stores.accessList.addUser(8996, CONTRACT, user(1)) + await stores.accessList.addUser(8996, CONTRACT, user(2)) + await stores.accessList.removeUserByTokenId(8996, CONTRACT, 1) + const list = await stores.accessList.retrieve(8996, CONTRACT) + expect(list.users.map((u: any) => u.tokenId)).to.deep.equal([2]) + expect(await stores.accessList.removeUserByTokenId(137, CONTRACT, 1)).to.equal(null) + }) + + it('finds the lists of a wallet, on all chains or on one', async () => { + await stores.accessList.addUser(8996, CONTRACT, user(1)) + await stores.accessList.addUser(137, CONTRACT, user(1)) + await stores.accessList.addUser(137, WALLET, user(1, CONTRACT)) + expect(await stores.accessList.searchByWallet(WALLET)).to.have.length(2) + const onChain = await stores.accessList.searchByWallet(WALLET.toUpperCase(), 137) + expect(onChain.map((list) => list.chainId)).to.deep.equal([137]) + expect(await stores.accessList.delete(137, CONTRACT)).to.deep.equal({ + id: `137-${CONTRACT.toLowerCase()}` + }) + expect(await stores.accessList.searchByWallet(WALLET, 137)).to.deep.equal([]) + }) + }) + + describe('logs', () => { + const DAY_MS = 24 * 60 * 60 * 1000 + const everything = () => [new Date(0), new Date(Date.now() + DAY_MS)] as const + + it('stores a log and reads it back with an id and a timestamp', async () => { + const inserted = await stores.logs.insertLog({ + level: 'info', + message: 'hello', + moduleName: 'HTTP' + }) + const log = await stores.logs.retrieveLog(inserted.id) + expect(log.message).to.equal('hello') + expect(log.id).to.equal(inserted.id) + expect(new Date(log.timestamp).getTime()).to.be.closeTo(Date.now(), 5000) + expect(await stores.logs.retrieveLog('none')).to.equal(null) + }) + + it('lists the logs, newest first, by module and level, in pages', async () => { + for (const [level, moduleName] of [ + ['info', 'HTTP'], + ['error', 'HTTP'], + ['info', 'P2P'] + ]) { + await stores.logs.insertLog({ + level, + moduleName, + message: `${moduleName}-${level}` + }) + await new Promise((resolve) => setTimeout(resolve, 2)) + } + const messages = async (...args: [number, string?, string?, number?]) => + (await stores.logs.retrieveMultipleLogs(...everything(), ...args)).map( + (log) => log.message + ) + expect(await messages(10)).to.deep.equal(['P2P-info', 'HTTP-error', 'HTTP-info']) + expect(await messages(2)).to.deep.equal(['P2P-info', 'HTTP-error']) + expect(await messages(10, 'http')).to.deep.equal(['HTTP-error', 'HTTP-info']) + expect(await messages(10, 'HTTP', 'error')).to.deep.equal(['HTTP-error']) + expect(await messages(2, undefined, undefined, 1)).to.deep.equal(['HTTP-info']) + expect(await stores.logs.getLogsCount()).to.equal(3) + }) + + it('deletes one log, and the logs that are older than the retention time', async () => { + const first = await stores.logs.insertLog({ level: 'info', message: 'first' }) + await stores.logs.insertLog({ level: 'info', message: 'second' }) + await stores.logs.delete(first.id) + expect(await stores.logs.getLogsCount()).to.equal(1) + expect(await stores.logs.deleteOldLogs()).to.equal(0) + await db.run('UPDATE logs SET "timestamp" = ?', ['2020-01-01T00:00:00.000Z']) + expect(await stores.logs.deleteOldLogs()).to.equal(1) + expect(await stores.logs.getLogsCount()).to.equal(0) + }) + }) + + describe('ddo', () => { + const copy = (ddo: Record) => structuredClone(ddo) + const all = { offset: 0, limit: 10 } + const byId: SortSpec[] = [{ paths: ['id'], order: 'asc' }] + const found = async (conditions: Condition[] = []) => + (await stores.ddo.find(conditions, byId, all)).results.map((ddo: any) => ddo.id) + + beforeEach(async () => { + await stores.ddo.create(copy(DDOExample)) + await stores.ddo.create(copy(ddov5)) + }) + + it('stores a valid DDO and reads it back', async () => { + const ddo = await stores.ddo.retrieve(DDOExample.id) + expect(ddo.id).to.equal(DDOExample.id) + expect(ddo.metadata).to.deep.equal(DDOExample.metadata) + expect(await stores.ddo.retrieve('did:op:none')).to.equal(null) + }) + + it('refuses a DDO that is not valid, and a DDO with an unknown version', async () => { + const noMetadata = { ...copy(DDOExample), id: 'did:op:invalid', metadata: {} } + expect(await stores.ddo.create(noMetadata)).to.equal(null) + expect(await stores.ddo.retrieve('did:op:invalid')).to.equal(null) + expect(() => stores.ddo.create({ id: 'did:op:x', version: '0.0.1' })).to.throw( + 'Unsupported DDO version: 0.0.1' + ) + }) + + it('finds the DDOs of all versions, with a page and the total', async () => { + expect(await found()).to.deep.equal([DDOExample.id, ddov5.id].sort()) + expect(await found([{ paths: ['chainId'], anyOf: [1] }])).to.deep.equal([]) + const page = await stores.ddo.find([], byId, { offset: 1, limit: 1 }) + expect(page.results).to.have.length(1) + expect(page.totalResults).to.equal(2) + }) + + it('gives the distinct tags of the DDOs that match', async () => { + expect(await stores.ddo.tags([], 10)).to.deep.equal([ + { tag: 'defi', count: 1 }, + { tag: 'index', count: 1 }, + { tag: 'tvl', count: 1 }, + { tag: 'white-papers', count: 1 } + ]) + expect( + await stores.ddo.tags([{ paths: ['chainId'], anyOf: [V5_CHAIN] }], 10) + ).to.deep.equal([{ tag: 'white-papers', count: 1 }]) + }) + + it('an update replaces the DDO', async () => { + const deprecated = copy(DDOExample) + deprecated.indexedMetadata = { ...deprecated.indexedMetadata } + deprecated.indexedMetadata.nft = { ...deprecated.indexedMetadata?.nft, state: 5 } + await stores.ddo.update(deprecated) + const state = [{ paths: ['indexedMetadata.nft.state'], anyOf: [5] }] + expect(await found(state)).to.deep.equal([DDOExample.id]) + }) + + it('deletes one DDO, and all the DDOs of a chain (v4 and v5 chain fields)', async () => { + expect(await stores.ddo.delete(ddov5.id)).to.deep.equal({ id: ddov5.id }) + expect(await stores.ddo.delete(ddov5.id)).to.equal(undefined) + expect(await stores.ddo.deleteAllAssetsFromChain(DDOExample.chainId)).to.equal(1) + expect(await found()).to.deep.equal([]) + }) + }) +}) diff --git a/src/test/unit/database/postgresClient.test.ts b/src/test/unit/database/postgresClient.test.ts new file mode 100644 index 000000000..85ea082e0 --- /dev/null +++ b/src/test/unit/database/postgresClient.test.ts @@ -0,0 +1,44 @@ +import { expect } from 'chai' +import { + connectionString, + numberPlaceholders +} from '../../../components/database/sql/PostgresClient.js' + +describe('PostgresClient', () => { + describe('numberPlaceholders', () => { + it('numbers the placeholders in their order', () => { + expect( + numberPlaceholders('SELECT * FROM t WHERE "a" = ? AND "b" IN (?, ?)') + ).to.equal('SELECT * FROM t WHERE "a" = $1 AND "b" IN ($2, $3)') + }) + + it('keeps a question mark that is in a text or in a name', () => { + expect( + numberPlaceholders(`SELECT '?', 'it''s ?', "why?" FROM t WHERE "a" = ?`) + ).to.equal(`SELECT '?', 'it''s ?', "why?" FROM t WHERE "a" = $1`) + }) + }) + + describe('connectionString', () => { + const config = (username: string, password: string) => + ({ + dbType: 'postgresql', + url: 'postgresql://db:5432/ocean', + username, + password + }) as const + + it('puts the user and the password into the URL', () => { + expect(connectionString(config('ocean', 'secret'))).to.equal( + 'postgresql://ocean:secret@db:5432/ocean' + ) + }) + + it('gives back the same password after the decoding that pg does', () => { + for (const password of ['ab%41cd', 'p@ss:w/rd#1?', 'ünï côde', '100%']) { + const url = new URL(connectionString(config('ocean', password))) + expect(decodeURIComponent(url.password)).to.equal(password) + } + }) + }) +}) diff --git a/src/test/unit/database/sqlDocuments.test.ts b/src/test/unit/database/sqlDocuments.test.ts new file mode 100644 index 000000000..91e04d6a5 --- /dev/null +++ b/src/test/unit/database/sqlDocuments.test.ts @@ -0,0 +1,448 @@ +import { expect } from 'chai' +import type { SqlClient } from '../../../components/database/sql/SqlClient.js' +import { makeSqlClient } from '../../utils/sqlClient.js' +import { + SqlDocuments, + flattenDocument +} from '../../../components/database/sql/SqlDocuments.js' +import type { Condition, SortSpec } from '../../../components/database/BaseDatabase.js' + +// Conditions, sort and paging on a SQL document table. + +const WEATHER = { + id: 'did:op:weather', + chainId: 8996, + nftAddress: '0xAAA', + metadata: { + name: 'Ocean Weather Data', + type: 'dataset', + tags: ['weather', 'Climate'], + created: '2026-01-10T00:00:00Z', + description: 'Hourly temperature of the North Sea' + }, + services: [ + { type: 'access', datatokenAddress: '0xD1' }, + { type: 'compute', datatokenAddress: '0xD2' } + ], + indexedMetadata: { + nft: { state: 0, owner: '0xOwnerA' }, + stats: [{ orders: 5, prices: [{ type: 'fixedrate', price: '10' }] }], + purgatory: { state: false }, + event: { block: 100 } + } +} +const CLASSIFIER = { + id: 'did:op:classifier', + chainId: 8996, + nftAddress: '0xBBB', + metadata: { + name: 'Image Classifier', + type: 'algorithm', + tags: ['ml', 'weather'], + created: '2026-02-10T00:00:00Z' + }, + services: [{ type: 'access', datatokenAddress: '0xD3' }], + indexedMetadata: { + nft: { state: 0, owner: '0xOwnerB' }, + stats: [{ orders: 12 }], + purgatory: { state: false }, + event: { block: 200 } + } +} +const ARCHIVE = { + id: 'did:op:archive', + chainId: 137, + nftAddress: '0xCCC', + metadata: { + name: 'Old weather archive', + type: 'dataset', + created: '2026-03-10T00:00:00Z' + }, + services: [{ type: 'access', datatokenAddress: '0xD4' }], + indexedMetadata: { + nft: { state: 5, owner: '0xOwnerA' }, + stats: [{ orders: 0 }], + purgatory: { state: true }, + event: { block: 300 } + } +} + +const ALL = { offset: 0, limit: 100 } + +describe('flattenDocument', () => { + it('gives one row per leaf value, with no array index in the path', () => { + const rows = flattenDocument({ a: { b: 'X' }, list: [{ n: 1 }, { n: 2.5 }], t: true }) + expect(rows).to.deep.equal([ + { path: 'a.b', str: 'x', num: null }, + { path: 'list.n', str: '1', num: 1 }, + { path: 'list.n', str: '2.5', num: 2.5 }, + { path: 't', str: 'true', num: null } + ]) + }) + + it('gives no row for null, and keeps the full text of a long string', () => { + const rows = flattenDocument({ none: null, long: 'A'.repeat(600) }) + expect(rows).to.have.length(1) + expect(rows[0].str).to.equal('a'.repeat(600)) + }) +}) + +describe('SQL documents', () => { + let db: SqlClient + let documents: SqlDocuments + + const ids = async (conditions: Condition[], sort: SortSpec[] = []) => + (await documents.find(conditions, sort, ALL)).results.map((doc) => doc.id) + const sorted = async (conditions: Condition[]) => (await ids(conditions)).sort() + const count = async () => + Number((await db.get('SELECT COUNT(*) AS "count" FROM ddo')).count) + + beforeEach(async () => { + db = await makeSqlClient() + documents = new SqlDocuments(db, 'ddo') + await documents.init() + await documents.put(WEATHER.id, WEATHER) + await documents.put(CLASSIFIER.id, CLASSIFIER) + await documents.put(ARCHIVE.id, ARCHIVE) + }) + afterEach(() => db.close()) + + describe('documents', () => { + it('reads a document back, and null for an unknown id', async () => { + expect(await documents.get(WEATHER.id)).to.deep.equal(WEATHER) + expect(await documents.get('none')).to.equal(null) + }) + + it('a second put replaces the document and its old field rows', async () => { + await documents.put(WEATHER.id, { ...WEATHER, chainId: 1 }) + expect(await sorted([{ paths: ['chainId'], anyOf: [8996] }])).to.deep.equal([ + CLASSIFIER.id + ]) + expect(await sorted([{ paths: ['chainId'], anyOf: [1] }])).to.deep.equal([ + WEATHER.id + ]) + expect(await count()).to.equal(3) + }) + + it('adds the id to a document that has no id field', async () => { + await documents.put('order-1', { consumer: '0xC' }) + expect(await documents.get('order-1')).to.deep.equal({ + consumer: '0xC', + id: 'order-1' + }) + expect(await ids([{ paths: ['id'], anyOf: ['order-1'] }])).to.deep.equal([ + 'order-1' + ]) + }) + + it('modify changes a document in one step, and creates it when absent', async () => { + await documents.modify('counter', (doc) => ({ count: (doc?.count ?? 0) + 1 })) + await documents.modify('counter', (doc) => ({ count: (doc?.count ?? 0) + 1 })) + expect((await documents.get('counter')).count).to.equal(2) + }) + + it('modifies of one document at the same time do not lose a change', async () => { + await documents.put('counter', { count: 0 }) + const increment = () => + documents.modify('counter', (doc) => ({ count: doc.count + 1 })) + await Promise.all(Array.from({ length: 20 }, increment)) + expect((await documents.get('counter')).count).to.equal(20) + }) + + it('modifies that make one document at the same time keep all changes', async () => { + const add = (item: number) => + documents.modify('list', (doc) => ({ items: [...(doc?.items ?? []), item] })) + await Promise.all(Array.from({ length: 20 }, (_, item) => add(item))) + expect((await documents.get('list')).items).to.have.length(20) + }) + + it('a modify that changes nothing leaves no document', async () => { + expect(await documents.modify('absent', () => null)).to.equal(null) + expect(await documents.get('absent')).to.equal(null) + expect(await count()).to.equal(3) + }) + + it('gives the documents in the order of the bytes of their ids', async () => { + for (const id of ['e1', 'e10', 'e-dup', 'E2']) { + await documents.put(id, { kind: 'event' }) + } + expect(await ids([{ paths: ['kind'], anyOf: ['event'] }])).to.deep.equal([ + 'E2', + 'e-dup', + 'e1', + 'e10' + ]) + }) + + it('deletes a document with its field rows', async () => { + expect(await documents.delete(WEATHER.id)).to.equal(true) + expect(await documents.delete(WEATHER.id)).to.equal(false) + expect(await sorted([])).to.deep.equal([ARCHIVE.id, CLASSIFIER.id]) + const rows = await db.all('SELECT * FROM ddo_fields WHERE "id" = ?', [WEATHER.id]) + expect(rows).to.deep.equal([]) + }) + + it('keeps a document whose text has a NUL character', async () => { + await documents.put('nul', { + name: 'a\u0000b', + 'k\u0000ey': 'v', + metadata: { 'k\u0000ey': 'w' } + }) + expect(await sorted([{ paths: ['name'], anyOf: ['ab'] }])).to.deep.equal(['nul']) + expect(await sorted([{ paths: ['metadata.key'], anyOf: ['w'] }])).to.deep.equal([ + 'nul' + ]) + expect((await documents.get('nul')).name).to.equal('a\u0000b') + }) + + it('deletes many documents with their field rows, more than one statement holds', async () => { + for (let i = 0; i < 450; i++) await documents.put(`bulk-${i}`, { kind: 'bulk' }) + expect( + await documents.deleteMatching([{ paths: ['kind'], anyOf: ['bulk'] }]) + ).to.equal(450) + expect(await count()).to.equal(3) + const left = await db.get( + 'SELECT COUNT(*) AS "n" FROM ddo_fields WHERE "id" LIKE ?', + ['bulk-%'] + ) + expect(Number(left.n)).to.equal(0) + }) + + it('deletes the documents that meet the conditions', async () => { + expect( + await documents.deleteMatching([{ paths: ['chainId'], anyOf: [8996] }]) + ).to.equal(2) + expect(await sorted([])).to.deep.equal([ARCHIVE.id]) + }) + }) + + describe('conditions', () => { + it('anyOf: a scalar, a number and a boolean, in any letter case', async () => { + expect( + await sorted([{ paths: ['metadata.type'], anyOf: ['DATASET'] }]) + ).to.deep.equal([ARCHIVE.id, WEATHER.id]) + expect(await sorted([{ paths: ['chainId'], anyOf: [137] }])).to.deep.equal([ + ARCHIVE.id + ]) + expect( + await sorted([{ paths: ['indexedMetadata.purgatory.state'], anyOf: [true] }]) + ).to.deep.equal([ARCHIVE.id]) + }) + + it('anyOf: a field in an array matches when one element matches', async () => { + expect( + await sorted([{ paths: ['services.type'], anyOf: ['compute'] }]) + ).to.deep.equal([WEATHER.id]) + expect( + await sorted([{ paths: ['metadata.tags'], anyOf: ['climate'] }]) + ).to.deep.equal([WEATHER.id]) + }) + + it('anyOf: one of the values, on one of the paths; no value matches nothing', async () => { + expect( + await sorted([{ paths: ['id'], anyOf: [WEATHER.id, ARCHIVE.id, 'did:op:none'] }]) + ).to.deep.equal([ARCHIVE.id, WEATHER.id]) + expect( + await sorted([{ paths: ['nftAddress', 'other.nftAddress'], anyOf: ['0xbbb'] }]) + ).to.deep.equal([CLASSIFIER.id]) + expect(await ids([{ paths: ['id'], anyOf: [] }])).to.deep.equal([]) + }) + + it('noneOf: a document without the field passes; no value excludes nothing', async () => { + expect( + await sorted([{ paths: ['indexedMetadata.nft.state'], noneOf: [5] }]) + ).to.deep.equal([CLASSIFIER.id, WEATHER.id]) + expect( + await sorted([{ paths: ['metadata.description'], noneOf: ['x'] }]) + ).to.deep.equal([ARCHIVE.id, CLASSIFIER.id, WEATHER.id]) + expect(await sorted([{ paths: ['id'], noneOf: [] }])).to.have.length(3) + }) + + it('range: numbers as numbers, ISO dates as text', async () => { + expect( + await sorted([{ paths: ['indexedMetadata.stats.orders'], range: { gt: 4 } }]) + ).to.deep.equal([CLASSIFIER.id, WEATHER.id]) + expect( + await sorted([ + { paths: ['metadata.created'], range: { gte: '2026-02-01T00:00:00.000Z' } } + ]) + ).to.deep.equal([ARCHIVE.id, CLASSIFIER.id]) + }) + + it('exists: a leaf field and an object field', async () => { + expect( + await sorted([{ paths: ['metadata.description'], exists: true }]) + ).to.deep.equal([WEATHER.id]) + expect(await sorted([{ paths: ['indexedMetadata'], exists: true }])).to.have.length( + 3 + ) + expect(await sorted([{ paths: ['nothing'], exists: true }])).to.deep.equal([]) + }) + + it('words: each word inside a value of one of the fields, in any letter case', async () => { + const text = ['metadata.name', 'metadata.description', 'metadata.tags'] + expect(await sorted([{ paths: text, words: ['WEATHER'] }])).to.deep.equal([ + ARCHIVE.id, + CLASSIFIER.id, + WEATHER.id + ]) + expect(await sorted([{ paths: text, words: ['weather', 'sea'] }])).to.deep.equal([ + WEATHER.id + ]) + expect(await sorted([{ paths: text, words: ['clas'] }])).to.deep.equal([ + CLASSIFIER.id + ]) + // a LIKE wildcard in a word is a plain character + expect(await sorted([{ paths: text, words: ['%'] }])).to.deep.equal([]) + }) + + it('a value longer than the index key: words after the key, and full equality', async () => { + const long = 'a'.repeat(600) + await documents.put('long', { metadata: { description: `${long} ocean` } }) + const description = ['metadata.description'] + expect(await ids([{ paths: description, words: ['ocean'] }])).to.deep.equal([ + 'long' + ]) + expect(await ids([{ paths: description, anyOf: [`${long} ocean`] }])).to.deep.equal( + ['long'] + ) + // the same first 512 characters are not the same value + expect(await ids([{ paths: description, anyOf: [long] }])).to.deep.equal([]) + }) + + it('either: one of the conditions holds; no condition matches nothing', async () => { + const either: Condition[] = [ + { paths: ['id'], anyOf: [ARCHIVE.id] }, + { paths: ['indexedMetadata.nft.owner'], anyOf: ['0xownerb'] } + ] + expect(await sorted([{ either }])).to.deep.equal([ARCHIVE.id, CLASSIFIER.id]) + expect(await ids([{ either: [] }])).to.deep.equal([]) + }) + + it('all conditions must hold', async () => { + expect( + await sorted([ + { paths: ['metadata.type'], anyOf: ['dataset'] }, + { paths: ['indexedMetadata.nft.state'], noneOf: [5] } + ]) + ).to.deep.equal([WEATHER.id]) + }) + }) + + describe('sort, paging and counts', () => { + const orders: SortSpec = { paths: ['indexedMetadata.stats.orders'], order: 'desc' } + + it('sorts a number field as a number, in the two directions', async () => { + expect(await ids([], [orders])).to.deep.equal([ + CLASSIFIER.id, + WEATHER.id, + ARCHIVE.id + ]) + expect(await ids([], [{ ...orders, order: 'asc' }])).to.deep.equal([ + ARCHIVE.id, + WEATHER.id, + CLASSIFIER.id + ]) + }) + + it('puts the documents without the field last', async () => { + const description: SortSpec = { paths: ['metadata.description'], order: 'asc' } + const found = await ids([], [description]) + expect(found[0]).to.equal(WEATHER.id) + }) + + it('gives one page and the number of all matches', async () => { + const page = await documents.find([], [orders], { offset: 1, limit: 1 }) + expect(page.results.map((doc) => doc.id)).to.deep.equal([WEATHER.id]) + expect(page.totalResults).to.equal(3) + const countOnly = await documents.find([], [], { offset: 0, limit: 0 }) + expect(countOnly).to.deep.equal({ results: [], totalResults: 3 }) + }) + + it('counts the distinct values of a field in the documents that match', async () => { + const tags = await documents.valueCounts(['metadata.tags'], [], 10) + expect(tags).to.deep.equal([ + { value: 'weather', count: 2 }, + { value: 'climate', count: 1 }, + { value: 'ml', count: 1 } + ]) + const ofDatasets = await documents.valueCounts( + ['metadata.tags'], + [{ paths: ['metadata.type'], anyOf: ['dataset'] }], + 1 + ) + expect(ofDatasets).to.deep.equal([{ value: 'climate', count: 1 }]) + }) + }) +}) + +// Differences that a comparison with Elasticsearch on the same documents found. +describe('SQL documents: values and order', () => { + let db: SqlClient + let documents: SqlDocuments + const LONG = 'x'.repeat(600) + + const ids = async (conditions: Condition[], sort: SortSpec[] = []) => + (await documents.find(conditions, sort, ALL)).results.map((doc) => doc.id) + + beforeEach(async () => { + db = await makeSqlClient() + documents = new SqlDocuments(db, 'ddo') + await documents.init() + await documents.put('cheap', { + name: '_x', + price: '2.5', + created: '2022-06-10T08:00:00Z' + }) + await documents.put('dear', { + name: 'Ärger', + price: '10', + created: '2023-01-05T08:00:00Z' + }) + await documents.put('free', { name: 'a', price: 0, license: LONG }) + }) + afterEach(() => db.close()) + + it('compares and sorts a number that is stored as text as a number', async () => { + // as text, '10' is below '2.5' + expect(await ids([{ paths: ['price'], range: { gte: 5 } }])).to.deep.equal(['dear']) + expect(await ids([{ paths: ['price'], range: { gt: '0', lt: '5' } }])).to.deep.equal([ + 'cheap' + ]) + expect(await ids([], [{ paths: ['price'], order: 'desc' }])).to.deep.equal([ + 'dear', + 'cheap', + 'free' + ]) + }) + + it('compares a date with a bound that holds only a year', async () => { + expect(await ids([{ paths: ['created'], range: { gte: '2023' } }])).to.deep.equal([ + 'dear' + ]) + }) + + it('finds a value that is longer than the text that the field table keeps', async () => { + expect(await ids([{ paths: ['license'], anyOf: [LONG] }])).to.deep.equal(['free']) + expect(await ids([{ paths: ['license'], words: [LONG] }])).to.deep.equal(['free']) + }) + + it('sorts and compares text by its bytes, the same on all databases', async () => { + // lower case: '_' (0x5f), 'a' (0x61), 'ä' (0xc3 0xa4) + expect(await ids([], [{ paths: ['name'], order: 'asc' }])).to.deep.equal([ + 'cheap', + 'free', + 'dear' + ]) + expect(await ids([{ paths: ['name'], range: { gte: 'a', lt: 'c' } }])).to.deep.equal([ + 'free' + ]) + }) + + it('gives no field row to a path that is longer than a field name', async () => { + const key = 'k'.repeat(200) + await documents.put('long-path', { [key]: 'value', name: 'b' }) + expect(await ids([{ paths: ['name'], anyOf: ['b'] }])).to.deep.equal(['long-path']) + expect(await ids([{ paths: [key], exists: true }])).to.deep.equal([]) + }) +}) diff --git a/src/test/unit/database/sqliteClient.test.ts b/src/test/unit/database/sqliteClient.test.ts new file mode 100644 index 000000000..930cd7653 --- /dev/null +++ b/src/test/unit/database/sqliteClient.test.ts @@ -0,0 +1,93 @@ +import { expect } from 'chai' +import { SqliteClient } from '../../../components/database/sql/SqliteClient.js' + +describe('SqliteClient', () => { + let db: SqliteClient + + beforeEach(async () => { + db = new SqliteClient(':memory:') + await db.exec('CREATE TABLE items ("name" TEXT, "count" BIGINT)') + }) + + afterEach(async () => { + await db.close() + }) + + const names = async () => + (await db.all<{ name: string }>('SELECT "name" FROM items')).map((row) => row.name) + + it('run returns the number of changed rows', async () => { + await db.run('INSERT INTO items VALUES (?, ?)', ['a', 1]) + await db.run('INSERT INTO items VALUES (?, ?)', ['b', 1]) + const { changes } = await db.run( + 'UPDATE items SET "count" = ? WHERE "count" = ?', + [2, 1] + ) + expect(changes).to.equal(2) + }) + + it('get returns undefined when no row matches', async () => { + expect(await db.get('SELECT * FROM items WHERE "name" = ?', ['none'])).to.equal( + undefined + ) + }) + + it('binds undefined as null and booleans as 0 and 1', async () => { + await db.run('INSERT INTO items VALUES (?, ?)', [undefined, true]) + expect(await db.get('SELECT * FROM items')).to.deep.equal({ name: null, count: 1 }) + }) + + it('rejects a failed statement instead of throwing', async () => { + let error: Error + await db.run('INSERT INTO no_such_table VALUES (1)').catch((err) => { + error = err + }) + expect(error?.message).to.match(/no such table/) + }) + + it('transaction commits the work', async () => { + await db.transaction(async (tx) => { + await tx.run('INSERT INTO items VALUES (?, ?)', ['a', 1]) + }) + expect(await names()).to.deep.equal(['a']) + }) + + it('transaction rolls back when the work throws', async () => { + let error: Error + await db + .transaction(async (tx) => { + await tx.run('INSERT INTO items VALUES (?, ?)', ['a', 1]) + throw new Error('stop') + }) + .catch((err) => { + error = err + }) + expect(error?.message).to.equal('stop') + expect(await names()).to.deep.equal([]) + }) + + it('two concurrent transactions do not interleave', async () => { + const insertPair = (prefix: string) => + db.transaction(async (tx) => { + await tx.run('INSERT INTO items VALUES (?, ?)', [`${prefix}1`, 1]) + // give the other transaction a chance to run between the two statements + await new Promise((resolve) => setTimeout(resolve, 5)) + await tx.run('INSERT INTO items VALUES (?, ?)', [`${prefix}2`, 1]) + }) + await Promise.all([insertPair('a'), insertPair('b')]) + expect(await names()).to.deep.equal(['a1', 'a2', 'b1', 'b2']) + }) + + it('a statement outside a transaction waits for the transaction to end', async () => { + const transaction = db.transaction(async (tx) => { + await tx.run('INSERT INTO items VALUES (?, ?)', ['a', 1]) + await new Promise((resolve) => setTimeout(resolve, 5)) + throw new Error('stop') + }) + const outside = db.run('INSERT INTO items VALUES (?, ?)', ['b', 1]) + await transaction.catch(() => {}) + await outside + // "b" is not in the rolled back transaction + expect(await names()).to.deep.equal(['b']) + }) +}) diff --git a/src/test/unit/database/sqliteUpgrade.test.ts b/src/test/unit/database/sqliteUpgrade.test.ts new file mode 100644 index 000000000..3886fdef4 --- /dev/null +++ b/src/test/unit/database/sqliteUpgrade.test.ts @@ -0,0 +1,294 @@ +import { expect } from 'chai' +import fs from 'fs' +import os from 'os' +import path from 'path' +import { DatabaseSync } from 'node:sqlite' +import { + createSqliteStores, + openSqliteClients, + upgradeLegacyTables +} from '../../../components/database/sql/SqliteStores.js' +import { + SqlStateStores, + buildSqlStateStores +} from '../../../components/database/sql/SqlStores.js' +import { HOUR_MS } from '../../../components/database/nodeMetricsAggregate.js' +import { ServiceStatusNumber } from '../../../@types/C2D/ServiceOnDemand.js' +import type { NodeMetricsSnapshot } from '../../../@types/nodeMetrics.js' + +// A node that updates from a release before the pluggable database must keep its state. +// The tables below are the ones that release made (copied from its code), with one row each. +const OWNER = '0xe2DD09d719Da89e5a3D0F2549c7E24566e947260' +const PLAIN_TOKEN = 'eyJhbGciOiJIUzI1NiJ9.payload.signature' +// an older release stored the end of a token as the client sent it +const EXPIRED_TOKEN = 'eyJhbGciOiJIUzI1NiJ9.expired.signature' +const HOUR = 500 * HOUR_MS +const METRICS = + 'cpuUsagePercent REAL DEFAULT 0, coresAllocated REAL DEFAULT 0, hostCores REAL DEFAULT 0,' + + ' throttledCount REAL DEFAULT 0, memUsedBytes REAL DEFAULT 0, memLimitBytes REAL DEFAULT 0,' + + ' hostFreeBytes REAL DEFAULT 0, hostTotalBytes REAL DEFAULT 0, diskUsedBytes REAL DEFAULT 0,' + + ' rxBytes REAL DEFAULT 0, txBytes REAL DEFAULT 0, jobsRunning REAL DEFAULT 0,' + + ' jobsRunningFree REAL DEFAULT 0, jobsQueued REAL DEFAULT 0, jobsQueuedFree REAL DEFAULT 0,' + + " sampledContainers REAL DEFAULT 0, gpu TEXT DEFAULT '[]', env TEXT DEFAULT '[]'" + +const blob = (value: unknown) => Buffer.from(JSON.stringify(value)) + +const LEGACY_FILES: Record = { + 'c2dDatabase.sqlite': [ + [ + `CREATE TABLE c2djobs (owner TEXT, did TEXT DEFAULT NULL, jobId TEXT PRIMARY KEY, + dateCreated TEXT, dateFinished TEXT DEFAULT NULL, status INTEGER, statusText TEXT, + results BLOB, inputDID TEXT DEFAULT NULL, algoDID TEXT DEFAULT NULL, + agreementId TEXT DEFAULT NULL, expireTimestamp INTEGER, + environment TEXT DEFAULT NULL, body BLOB)`, + [] + ], + [ + `INSERT INTO c2djobs (owner, jobId, dateCreated, status, statusText, inputDID, + agreementId, expireTimestamp, environment, body) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + OWNER, + 'job-1', + '1785760000.5', + 40, + 'Running algorithm', + 'did:op:1__,__did:op:2', + '0xagreement', + 3600, + 'env-1', + blob({ clusterHash: 'cluster-1', isFree: true }) + ] + ], + [ + `CREATE TABLE service_jobs (serviceId TEXT PRIMARY KEY, owner TEXT, clusterHash TEXT, + status INTEGER, expiresAt INTEGER, dateCreated TEXT, body BLOB)`, + [] + ], + [ + 'INSERT INTO service_jobs VALUES (?, ?, ?, ?, ?, ?, ?)', + [ + 'svc-1', + OWNER, + 'cluster-1', + ServiceStatusNumber.Expired, + 1000, + '2026-01-01T00:00:00.000Z', + blob({ + serviceId: 'svc-1', + owner: OWNER, + clusterHash: 'cluster-1', + status: ServiceStatusNumber.Expired, + expiresAt: 1000, + outputArchives: [{ id: 'archive-1' }] + }) + ] + ] + ], + 'persistentStorage.sqlite': [ + [ + `CREATE TABLE persistent_storage_buckets (bucketId TEXT PRIMARY KEY, + owner TEXT NOT NULL, accessListJson TEXT NOT NULL, createdAt INTEGER NOT NULL)`, + [] + ], + [ + 'INSERT INTO persistent_storage_buckets VALUES (?, ?, ?, ?)', + ['b1', OWNER, '[]', 100] + ] + ], + 'nonceDatabase.sqlite': [ + ['CREATE TABLE nonce (id TEXT PRIMARY KEY, nonce INTEGER)', []], + ['INSERT INTO nonce VALUES (?, ?)', [OWNER, 7]] + ], + 'config.sqlite': [ + ['CREATE TABLE config (key TEXT NOT NULL PRIMARY KEY, value TEXT)', []], + ['INSERT INTO config VALUES (?, ?)', ['version', '5.0.0']] + ], + 'authTokenDatabase.sqlite': [ + [ + `CREATE TABLE authTokens (token TEXT PRIMARY KEY, address TEXT NOT NULL, + createdAt DATETIME DEFAULT CURRENT_TIMESTAMP, validUntil DATETIME, + isValid BOOLEAN DEFAULT TRUE)`, + [] + ], + [ + 'INSERT INTO authTokens (token, address, createdAt, validUntil) VALUES (?, ?, ?, ?)', + [PLAIN_TOKEN, OWNER, 1000, null] + ], + [ + 'INSERT INTO authTokens (token, address, createdAt, validUntil) VALUES (?, ?, ?, ?)', + [EXPIRED_TOKEN, OWNER, 1000, '2020-01-01T00:00:00.000Z'] + ] + ], + 'nodeMetrics.sqlite': [ + [ + `CREATE TABLE node_metrics_samples (id INTEGER PRIMARY KEY AUTOINCREMENT, + collectedAt INTEGER NOT NULL, ${METRICS})`, + [] + ], + [ + 'INSERT INTO node_metrics_samples (collectedAt, cpuUsagePercent) VALUES (?, ?)', + [HOUR + 1000, 40] + ], + [ + `CREATE TABLE node_metrics_hourly (hourStart INTEGER PRIMARY KEY, + sampleCount INTEGER DEFAULT 0, ${METRICS})`, + [] + ], + [ + 'INSERT INTO node_metrics_hourly (hourStart, sampleCount, cpuUsagePercent) VALUES (?, ?, ?)', + [HOUR - HOUR_MS, 60, 15] + ], + // an hourly row for the hour that still has a sample: the next roll-up replaces it + [ + 'INSERT INTO node_metrics_hourly (hourStart, sampleCount, cpuUsagePercent) VALUES (?, ?, ?)', + [HOUR, 1, 99] + ] + ] +} + +function writeLegacyFiles(dir: string): void { + for (const [file, statements] of Object.entries(LEGACY_FILES)) { + const db = new DatabaseSync(path.join(dir, file)) + for (const [sql, params] of statements) { + db.prepare(sql).run(...(params as any[])) + } + db.close() + } +} + +describe('SQLite stores on the databases folder of an older release', () => { + let dir: string + let stores: SqlStateStores + let closeAll: () => Promise + + // opens the folder the way the node does at startup + async function openStores(): Promise { + const clients = openSqliteClients(dir) + await upgradeLegacyTables(clients) + stores = await buildSqlStateStores({ dbType: 'sqlite' }, (group) => clients[group]) + closeAll = async () => { + await Promise.all(Object.values(clients).map((client) => client.close())) + } + } + + before(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ocean-legacy-db-')) + writeLegacyFiles(dir) + await openStores() + }) + + after(async () => { + await closeAll() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('keeps a compute job, with its body and its input DIDs', async () => { + const [job] = await stores.c2d.getJob('job-1') + expect(job.owner).to.equal(OWNER) + expect(job.dateCreated).to.equal('1785760000.5') + expect(job.inputDID).to.deep.equal(['did:op:1', 'did:op:2']) + expect(job.maxJobDuration).to.equal(3600) + expect(job.clusterHash).to.equal('cluster-1') + expect(job.isFree).to.equal(true) + }) + + it('keeps a service job and counts its output archives', async () => { + const [service] = await stores.c2d.getServiceJob('svc-1') + expect(service.owner).to.equal(OWNER) + const toClean = await stores.c2d.getExpiredServiceJobsBefore(2000) + expect(toClean.map((job) => job.serviceId)).to.deep.equal(['svc-1']) + }) + + it('keeps a bucket, a nonce and a config value', async () => { + expect(await stores.buckets.getBucket('b1')).to.deep.equal({ + bucketId: 'b1', + owner: OWNER, + accessListJson: '[]', + createdAt: 100, + label: null + }) + expect((await stores.nonce.retrieve(OWNER)).nonce).to.equal(7) + expect(await stores.config.retrieveValue('version')).to.deep.equal({ value: '5.0.0' }) + }) + + it('keeps an auth token valid, and stores its hash in place of the token', async () => { + const entry = await stores.authToken.validateToken(PLAIN_TOKEN) + expect(entry.address).to.equal(OWNER) + expect(entry.createdAt).to.equal(1000) + const db = new DatabaseSync(path.join(dir, 'authTokenDatabase.sqlite')) + const keys = db.prepare('SELECT token FROM authTokens').all() + db.close() + expect(keys).to.have.length(2) + for (const key of keys) { + expect(key.token).to.match(/^0x[0-9a-f]{64}$/) + } + }) + + it('keeps a token expired when its end is a date text', async () => { + expect(await stores.authToken.validateToken(EXPIRED_TOKEN)).to.equal(null) + }) + + it('shows the node metrics history before the node writes a sample', async () => { + const hourly = await stores.nodeMetrics.getHourly('node-a', 0, HOUR, 10) + expect(hourly.map((h) => h.hourStart)).to.deep.equal([HOUR - HOUR_MS, HOUR]) + expect( + await stores.nodeMetrics.getPendingSampleHours('node-a', HOUR + HOUR_MS) + ).to.deep.equal([HOUR]) + }) + + it('adds a new sample to the node metrics history of the older release', async () => { + const snapshot = { + collectedAt: HOUR + 2000, + hasAggregate: true, + cpu: { usagePercent: 20, coresAllocated: 0, hostCores: 0, throttledCount: 0 }, + memory: { usedBytes: 0, limitBytes: 0, hostFreeBytes: 0, hostTotalBytes: 0 }, + disk: { usedBytes: 0 }, + network: { rxBytes: 0, txBytes: 0 }, + jobs: { running: 0, runningFree: 0, queued: 0, queuedFree: 0 }, + gpu: [], + env: [], + meta: { sampledContainers: 0 } + } as unknown as NodeMetricsSnapshot + await stores.nodeMetrics.insertSample('node-a', snapshot) + expect(await stores.nodeMetrics.rollupHour('node-a', HOUR)).to.equal(true) + const hourly = await stores.nodeMetrics.getHourly('node-a', 0, HOUR, 10) + expect( + hourly.map((h) => [h.hourStart, h.sampleCount, h.cpu.usagePercent]) + ).to.deep.equal([ + [HOUR - HOUR_MS, 60, 15], + // the old sample (40) and the new one (20) + [HOUR, 2, 30] + ]) + }) + + it('counts the output archives at the next start when an upgrade did not end', async () => { + await closeAll() + const db = new DatabaseSync(path.join(dir, 'c2dDatabase.sqlite')) + db.exec('UPDATE service_jobs SET "outputArchiveCount" = NULL') + db.close() + await openStores() + const toClean = await stores.c2d.getExpiredServiceJobsBefore(2000) + expect(toClean.map((job) => job.serviceId)).to.deep.equal(['svc-1']) + }) + + it('closes the files when the start fails', async () => { + const bad = fs.mkdtempSync(path.join(os.tmpdir(), 'ocean-bad-db-')) + fs.writeFileSync(path.join(bad, 'nonceDatabase.sqlite'), 'this is not a database') + for (let attempt = 0; attempt < 3; attempt++) { + let error: Error + await createSqliteStores({ dbType: 'sqlite' }, bad).catch((err) => { + error = err + }) + expect(error).to.be.instanceOf(Error) + } + fs.rmSync(bad, { recursive: true, force: true }) + }) + + it('opens the folder a second time with the same result', async () => { + await closeAll() + await openStores() + expect((await stores.authToken.validateToken(PLAIN_TOKEN)).address).to.equal(OWNER) + expect(await stores.c2d.getJob('job-1')).to.have.length(1) + expect(await stores.nodeMetrics.getHourly('node-a', 0, HOUR, 10)).to.have.length(2) + }) +}) diff --git a/src/test/unit/database/stateStores.test.ts b/src/test/unit/database/stateStores.test.ts new file mode 100644 index 000000000..bd38d47f7 --- /dev/null +++ b/src/test/unit/database/stateStores.test.ts @@ -0,0 +1,649 @@ +import { expect } from 'chai' +import type { SqlClient } from '../../../components/database/sql/SqlClient.js' +import { SQL_DB_CONFIG, makeSqlClient } from '../../utils/sqlClient.js' +import { baseSnapshot } from '../../utils/utils.js' +import { SqlNonceDatabase } from '../../../components/database/sql/SqlNonce.js' +import { SqlConfigDatabase } from '../../../components/database/sql/SqlConfig.js' +import { SqlAuthToken } from '../../../components/database/sql/SqlAuthToken.js' +import { AuthTokenDatabase } from '../../../components/database/AuthTokenDatabase.js' +import { SqlBucketRegistry } from '../../../components/database/sql/SqlBucketRegistry.js' +import { SqlNodeMetrics } from '../../../components/database/sql/SqlNodeMetrics.js' +import { SqlCompute } from '../../../components/database/sql/SqlCompute.js' +import { HOUR_MS } from '../../../components/database/nodeMetricsAggregate.js' +import { + C2DStatusNumber, + C2DStatusText, + type DBComputeJob +} from '../../../@types/C2D/C2D.js' +import { + ServiceStatusNumber, + type ServiceJob +} from '../../../@types/C2D/ServiceOnDemand.js' + +// The contract of the state stores on a SQL database. Each test gets a new, empty database. + +const OWNER = '0xe2DD09d719Da89e5a3D0F2549c7E24566e947260' +const OTHER_OWNER = '0x0000000000000000000000000000000000000001' + +describe('SQL nonce store', () => { + let db: SqlClient + let nonces: SqlNonceDatabase + + beforeEach(async () => { + db = await makeSqlClient() + nonces = new SqlNonceDatabase(SQL_DB_CONFIG, db) + await nonces.init() + }) + afterEach(() => db.close()) + + it('creates a nonce and reads it back under the checksummed address', async () => { + await nonces.create(OWNER.toLowerCase(), 5) + expect(await nonces.retrieve(OWNER)).to.deep.equal({ id: OWNER, nonce: 5 }) + }) + + it('returns a null nonce for an unknown address', async () => { + expect(await nonces.retrieve(OWNER)).to.deep.equal({ id: OWNER, nonce: null }) + }) + + it('reads the highest nonce of all casings of the address', async () => { + await nonces.create(OWNER, 5) + // a row from before the addresses were normalized + await db.run('INSERT INTO nonce ("id", "nonce") VALUES (?, ?)', [ + OWNER.toLowerCase(), + 9 + ]) + expect((await nonces.retrieve(OWNER)).nonce).to.equal(9) + }) + + it('updates and deletes a nonce', async () => { + await nonces.create(OWNER, 5) + await nonces.update(OWNER, 6) + // a second request with the same nonce, or an older one, does not pass + expect(await nonces.update(OWNER, 6)).to.equal(null) + expect(await nonces.update(OWNER, 4)).to.equal(null) + expect(await nonces.delete(OWNER)).to.deep.equal({ id: OWNER, nonce: 6 }) + expect(await nonces.delete(OWNER)).to.deep.equal({ id: OWNER, nonce: null }) + }) +}) + +describe('SQL config store', () => { + let db: SqlClient + let config: SqlConfigDatabase + + beforeEach(async () => { + db = await makeSqlClient() + config = new SqlConfigDatabase(db) + await config.init() + }) + afterEach(() => db.close()) + + it('returns a null value for an absent key', async () => { + expect(await config.retrieveValue('version')).to.deep.equal({ value: null }) + }) + + it('creates a value and replaces it', async () => { + expect(await config.createOrUpdateConfig('version', '1')).to.deep.equal({ + key: 'version', + value: '1' + }) + await config.createOrUpdateConfig('version', '2') + expect(await config.retrieveValue('version')).to.deep.equal({ value: '2' }) + }) +}) + +describe('SQL auth token store', () => { + let db: SqlClient + let tokens: SqlAuthToken + const now = Date.now() + + beforeEach(async () => { + db = await makeSqlClient() + tokens = new SqlAuthToken(db) + await tokens.init() + }) + afterEach(() => db.close()) + + it('validates a token and returns its fields as numbers and a boolean', async () => { + await tokens.createToken('id-1', OWNER, now, now + HOUR_MS, '8996') + expect(await tokens.validateTokenEntry('id-1')).to.deep.equal({ + address: OWNER, + createdAt: now, + validUntil: now + HOUR_MS, + isValid: true, + chainId: '8996' + }) + }) + + it('keeps a chain id that comes as a number as its text', async () => { + await tokens.createToken('id-1', OWNER, now, null, 8996 as any) + expect((await tokens.validateTokenEntry('id-1')).chainId).to.equal('8996') + }) + + it('validates a token that has no end date', async () => { + await tokens.createToken('id-1', OWNER, now, null) + const entry = await tokens.validateTokenEntry('id-1') + expect(entry.validUntil).to.equal(null) + expect(entry.chainId).to.equal(null) + }) + + it('returns null for an unknown, an invalidated and an expired token', async () => { + await tokens.createToken('invalidated', OWNER, now, null) + await tokens.invalidateTokenEntry('invalidated') + await tokens.createToken('expired', OWNER, now - 2 * HOUR_MS, now - HOUR_MS) + expect(await tokens.validateTokenEntry('unknown')).to.equal(null) + expect(await tokens.validateTokenEntry('invalidated')).to.equal(null) + expect(await tokens.validateTokenEntry('expired')).to.equal(null) + }) + + it('takes the end of a token as a number or as a date, and refuses other values', async () => { + const database = new AuthTokenDatabase(SQL_DB_CONFIG, tokens) + const end = now + HOUR_MS + await database.createToken('number', OWNER, end + 0.5, now) + await database.createToken('text', OWNER, new Date(end).toISOString() as any, now) + await database.createToken('past', OWNER, '2020-01-01T00:00:00Z' as any, now) + expect((await database.validateToken('number')).validUntil).to.equal(end) + expect((await database.validateToken('text')).validUntil).to.equal(end) + expect(await database.validateToken('past')).to.equal(null) + let error: Error + await database.createToken('bad', OWNER, 'tomorrow' as any, now).catch((err) => { + error = err + }) + expect(error?.message).to.contain('validUntil') + }) + + it('deletes the expired and the invalidated tokens only', async () => { + await tokens.createToken('valid', OWNER, now, now + HOUR_MS) + await tokens.createToken('forever', OWNER, now, null) + await tokens.createToken('expired', OWNER, now - 2 * HOUR_MS, now - HOUR_MS) + await tokens.createToken('invalidated', OWNER, now, null) + await tokens.invalidateTokenEntry('invalidated') + expect(await tokens.deleteExpiredTokens(now)).to.equal(2) + expect(await tokens.validateTokenEntry('valid')).to.not.equal(null) + expect(await tokens.validateTokenEntry('forever')).to.not.equal(null) + }) +}) + +describe('SQL bucket registry', () => { + let db: SqlClient + let buckets: SqlBucketRegistry + + beforeEach(async () => { + db = await makeSqlClient() + buckets = new SqlBucketRegistry(db) + await buckets.init() + }) + afterEach(() => db.close()) + + it('creates a bucket and reads it back', async () => { + await buckets.upsertBucket('b1', OWNER, '[]', 100, null) + expect(await buckets.getBucket('b1')).to.deep.equal({ + bucketId: 'b1', + owner: OWNER, + accessListJson: '[]', + createdAt: 100, + label: null + }) + expect(await buckets.getBucket('none')).to.equal(null) + }) + + it('a second create changes the access list only', async () => { + await buckets.upsertBucket('b1', OWNER, '[]', 100, 'first') + await buckets.upsertBucket('b1', OTHER_OWNER, '[{"8996":["0x1"]}]', 200, 'second') + expect(await buckets.getBucket('b1')).to.deep.equal({ + bucketId: 'b1', + owner: OWNER, + accessListJson: '[{"8996":["0x1"]}]', + createdAt: 100, + label: 'first' + }) + }) + + it('lists the buckets of an owner, oldest first', async () => { + await buckets.upsertBucket('new', OWNER, '[]', 200, null) + await buckets.upsertBucket('old', OWNER, '[]', 100, null) + await buckets.upsertBucket('other', OTHER_OWNER, '[]', 150, null) + const ids = (await buckets.listBucketsByOwner(OWNER)).map((b) => b.bucketId) + expect(ids).to.deep.equal(['old', 'new']) + }) + + it('changes the label for the owner only, and deletes a bucket', async () => { + await buckets.upsertBucket('b1', OWNER, '[]', 100, null) + expect(await buckets.updateBucketLabel('b1', OTHER_OWNER, 'x')).to.equal(false) + expect(await buckets.updateBucketLabel('b1', OWNER, 'mine')).to.equal(true) + expect((await buckets.getBucket('b1')).label).to.equal('mine') + expect(await buckets.deleteBucket('b1')).to.equal(true) + expect(await buckets.deleteBucket('b1')).to.equal(false) + }) +}) + +describe('SQL node metrics store', () => { + let db: SqlClient + let metrics: SqlNodeMetrics + const HOUR = 500 * HOUR_MS + + const snapshot = (collectedAt: number, usagePercent: number) => + baseSnapshot({ collectedAt, cpu: { ...baseSnapshot().cpu, usagePercent } }) + + beforeEach(async () => { + db = await makeSqlClient() + metrics = new SqlNodeMetrics(db) + await metrics.init() + await metrics.insertSample('a', snapshot(HOUR + 1000, 10)) + await metrics.insertSample('a', snapshot(HOUR + 2000, 30)) + await metrics.insertSample('b', snapshot(HOUR + 3000, 50)) + }) + afterEach(() => db.close()) + + it('lists the hours that have samples before a given hour', async () => { + expect(await metrics.getPendingSampleHours('a', HOUR + HOUR_MS)).to.deep.equal([HOUR]) + expect(await metrics.getPendingSampleHours('a', HOUR)).to.deep.equal([]) + }) + + it('gives the average of an hour that is not rolled up, marked partial', async () => { + const partial = await metrics.getPartialHour('a', HOUR) + expect(partial.partial).to.equal(true) + expect(partial.sampleCount).to.equal(2) + expect(partial.cpu.usagePercent).to.equal(20) + expect(await metrics.getPartialHour('a', HOUR + HOUR_MS)).to.equal(null) + }) + + it('rolls an hour up into one hourly row and removes its samples', async () => { + expect(await metrics.rollupHour('a', HOUR)).to.equal(true) + expect(await metrics.rollupHour('a', HOUR)).to.equal(false) + const hourly = await metrics.getHourly('a', HOUR, HOUR, 10) + expect(hourly).to.have.length(1) + expect(hourly[0].hourStart).to.equal(HOUR) + expect(hourly[0].sampleCount).to.equal(2) + expect(hourly[0].cpu.usagePercent).to.equal(20) + // the samples and the hourly rows of another instance stay apart + expect((await metrics.getPartialHour('b', HOUR)).sampleCount).to.equal(1) + expect(await metrics.getHourly('b', HOUR, HOUR, 10)).to.deep.equal([]) + }) + + it('a second roll-up of the same hour replaces the hourly row', async () => { + await metrics.rollupHour('a', HOUR) + await metrics.insertSample('a', snapshot(HOUR + 4000, 70)) + await metrics.rollupHour('a', HOUR) + const hourly = await metrics.getHourly('a', HOUR, HOUR, 10) + expect(hourly).to.have.length(1) + expect(hourly[0].cpu.usagePercent).to.equal(70) + }) + + it('a roll-up counts the samples that it deletes, also when samples come at that time', async () => { + const late = Array.from({ length: 10 }, (_, i) => + metrics.insertSample('a', snapshot(HOUR + 5000 + i, 10)) + ) + await Promise.all([metrics.rollupHour('a', HOUR), ...late]) + const [hourly] = await metrics.getHourly('a', HOUR, HOUR, 10) + const left = (await metrics.getPartialHour('a', HOUR))?.sampleCount ?? 0 + // the two samples of the setup and the ten late ones: counted or still there + expect(hourly.sampleCount + left).to.equal(12) + }) + + it('one of the roll-ups of an hour that run at the same time does the work', async () => { + const done = await Promise.all( + Array.from({ length: 6 }, () => metrics.rollupHour('a', HOUR)) + ) + expect(done.filter(Boolean)).to.have.length(1) + expect((await metrics.getHourly('a', HOUR, HOUR, 10))[0].sampleCount).to.equal(2) + }) + + it('purges old samples and old hourly rows of one instance', async () => { + await metrics.rollupHour('b', HOUR) + expect(await metrics.purgeOldSamples('a', HOUR + 1500)).to.equal(1) + expect(await metrics.purgeOldSamples('a', HOUR + HOUR_MS)).to.equal(1) + expect(await metrics.deleteHourlyOlderThan('a', HOUR + HOUR_MS)).to.equal(0) + expect(await metrics.deleteHourlyOlderThan('b', HOUR + HOUR_MS)).to.equal(1) + }) +}) + +describe('SQL compute store', () => { + let db: SqlClient + let compute: SqlCompute + + const job = (overrides: Partial): DBComputeJob => + ({ + owner: OWNER, + did: null, + jobId: 'job-1', + dateCreated: '1785760000.5', + dateFinished: null, + status: C2DStatusNumber.RunningAlgorithm, + statusText: C2DStatusText.RunningAlgorithm, + inputDID: ['did:op:1', 'did:op:2'], + algoDID: 'did:op:algo', + agreementId: '0xagreement', + maxJobDuration: 3600, + environment: 'env-1', + clusterHash: 'cluster-1', + algorithm: { documentId: 'did:op:algo', serviceId: '0xabc' }, + assets: [{ documentId: 'did:op:1', serviceId: '0xdef' }], + resources: [{ id: 'cpu', amount: 2 }], + isFree: false, + ...overrides + }) as unknown as DBComputeJob + + const service = (overrides: Partial): ServiceJob => + ({ + serviceId: 'svc-1', + owner: OWNER, + clusterHash: 'cluster-1', + status: ServiceStatusNumber.Running, + expiresAt: Date.now() + HOUR_MS, + dateCreated: '2026-11-16T10:00:00.000Z', + containerId: 'container-1', + outputArchives: [], + ...overrides + }) as unknown as ServiceJob + + const expected = (job: ServiceJob) => ({ + owner: job.owner, + clusterHash: job.clusterHash, + status: job.status, + containerId: job.containerId + }) + + const ids = (jobs: DBComputeJob[]) => jobs.map((j) => j.jobId) + const serviceIds = (jobs: ServiceJob[]) => jobs.map((j) => j.serviceId).sort() + + beforeEach(async () => { + db = await makeSqlClient() + compute = new SqlCompute(db) + await compute.init() + }) + afterEach(() => db.close()) + + describe('jobs', () => { + it('creates a job and reads all its fields back', async () => { + expect(await compute.newJob(job({}))).to.equal('job-1') + const [read] = await compute.getJob('job-1') + expect(read.owner).to.equal(OWNER) + expect(read.dateCreated).to.equal('1785760000.5') + expect(read.dateFinished).to.equal(null) + expect(read.status).to.equal(C2DStatusNumber.RunningAlgorithm) + expect(read.inputDID).to.deep.equal(['did:op:1', 'did:op:2']) + expect(read.maxJobDuration).to.equal(3600) + expect(read.clusterHash).to.equal('cluster-1') + expect(read.resources).to.deep.equal([{ id: 'cpu', amount: 2 }]) + expect(read.jobIdHash).to.be.a('string') + expect(read).to.not.have.property('body') + expect(read).to.not.have.property('expireTimestamp') + }) + + it('finds a job by agreement id with and without 0x, and by owner in any case', async () => { + await compute.newJob(job({})) + expect(ids(await compute.getJob(undefined, 'agreement'))).to.deep.equal(['job-1']) + expect(ids(await compute.getJob(undefined, '0xagreement'))).to.deep.equal(['job-1']) + expect( + ids(await compute.getJob(undefined, undefined, OWNER.toLowerCase())) + ).to.deep.equal(['job-1']) + expect(await compute.getJob('job-1', undefined, OTHER_OWNER)).to.deep.equal([]) + expect(await compute.getJob('none')).to.deep.equal([]) + }) + + it('updates a job and returns the number of updated jobs', async () => { + await compute.newJob(job({})) + const finished = job({ + status: C2DStatusNumber.JobFinished, + statusText: C2DStatusText.JobFinished, + dateFinished: '1785760100', + isRunning: false + }) + expect(await compute.updateJob(finished)).to.equal(1) + const [read] = await compute.getJob('job-1') + expect(read.status).to.equal(C2DStatusNumber.JobFinished) + expect(read.dateFinished).to.equal('1785760100') + expect(await compute.updateJob(job({ jobId: 'none' }))).to.equal(0) + }) + + it('lists the running jobs, oldest first, by engine and environment', async () => { + await compute.newJob(job({ jobId: 'new', dateCreated: '1785760002' })) + await compute.newJob(job({ jobId: 'old', dateCreated: '999999999.5' })) + await compute.newJob(job({ jobId: 'other-env', environment: 'env-2' })) + await compute.newJob(job({ jobId: 'done' })) + await compute.updateJob(job({ jobId: 'done', dateFinished: '1785760100' })) + expect(ids(await compute.getRunningJobs('cluster-1', 'env-1'))).to.deep.equal([ + 'old', + 'new' + ]) + expect(await compute.getRunningJobs('cluster-2')).to.deep.equal([]) + expect(ids(await compute.getFinishedJobs(['env-1']))).to.deep.equal(['done']) + expect(await compute.getFinishedJobs(['env-2'])).to.deep.equal([]) + }) + + it('filters the job list by time as a number, by owner, status and environment', async () => { + const finished = (jobId: string, dateFinished: string, owner = OWNER) => + compute + .newJob(job({ jobId, owner, dateCreated: dateFinished })) + .then(() => + compute.updateJob( + job({ jobId, owner, dateFinished, status: C2DStatusNumber.JobFinished }) + ) + ) + // as text, '999999999.5' is above '1785760660'; as a number it is below + await finished('nine-digit', '999999999.5') + await finished('after', '1785760660.961') + await finished('other-owner', '1785760700', OTHER_OWNER) + await compute.newJob(job({ jobId: 'running', dateCreated: '1785760800' })) + + expect(ids(await compute.getJobs(undefined, 1785760660))).to.deep.equal([ + 'other-owner', + 'after' + ]) + expect( + ids(await compute.getJobs(undefined, undefined, [OWNER.toLowerCase()])) + ).to.deep.equal(['running', 'after', 'nine-digit']) + expect( + ids( + await compute.getJobs( + undefined, + undefined, + undefined, + C2DStatusNumber.JobStarted + ) + ) + ).to.deep.equal([]) + expect( + ids(await compute.getJobs(['env-1'], 1785760000, undefined, undefined, true)) + ).to.deep.equal(['running']) + expect( + ids(await compute.getJobsByStatus(['env-1'], [C2DStatusNumber.RunningAlgorithm])) + ).to.deep.equal(['running']) + }) + + it('refuses a date that is not a number: SQL compares the dates as numbers', async () => { + let error: Error + await compute.newJob(job({ dateCreated: 'yesterday' })).catch((err) => { + error = err + }) + expect(error?.message).to.contain('Unix seconds') + await compute.newJob(job({ dateCreated: '' })) + expect(Number((await compute.getJob('job-1'))[0].dateCreated)).to.be.closeTo( + Date.now() / 1000, + 60 + ) + }) + + it('deletes a job', async () => { + await compute.newJob(job({})) + expect(await compute.deleteJob('job-1')).to.equal(true) + expect(await compute.deleteJob('job-1')).to.equal(false) + }) + }) + + describe('images', () => { + it('records the use of an image and lists the images that are not used for some days', async () => { + await compute.updateImage('image:1') + await compute.updateImage('image:1') + expect(await compute.getOldImages(7)).to.deep.equal([]) + // a negative retention puts the limit in the future + expect(await compute.getOldImages(-1)).to.deep.equal(['image:1']) + await compute.deleteImage('image:1') + expect(await compute.getOldImages(-1)).to.deep.equal([]) + }) + }) + + describe('service jobs', () => { + it('creates a service job and finds it by id and by owner in any case', async () => { + await compute.newServiceJob(service({})) + const [read] = await compute.getServiceJob('svc-1') + expect(read.serviceId).to.equal('svc-1') + expect(read.containerId).to.equal('container-1') + expect( + serviceIds(await compute.getServiceJob(undefined, OWNER.toLowerCase())) + ).to.deep.equal(['svc-1']) + expect(await compute.getServiceJob('svc-1', OTHER_OWNER)).to.deep.equal([]) + }) + + it('updates a service job and returns the number of updated jobs', async () => { + await compute.newServiceJob(service({})) + const stopped = service({ status: ServiceStatusNumber.Stopped }) + expect(await compute.updateServiceJob(stopped)).to.equal(1) + expect((await compute.getServiceJob('svc-1'))[0].status).to.equal( + ServiceStatusNumber.Stopped + ) + expect(await compute.updateServiceJob(service({ serviceId: 'none' }))).to.equal(0) + }) + + it('adds metrics and readiness only when the job did not change', async () => { + const running = service({}) + await compute.newServiceJob(running) + const metrics: any = { cpuPercent: 12 } + const readiness: any = { ready: true } + expect( + await compute.updateServiceJobMetrics('svc-1', expected(running), metrics) + ).to.equal(true) + expect( + await compute.updateServiceJobReadiness('svc-1', expected(running), readiness) + ).to.equal(true) + const [read] = await compute.getServiceJob('svc-1') + expect(read.runtimeMetrics).to.deep.equal(metrics) + expect(read.readiness).to.deep.equal(readiness) + + await compute.updateServiceJob(service({ status: ServiceStatusNumber.Stopping })) + expect( + await compute.updateServiceJobMetrics('svc-1', expected(running), metrics) + ).to.equal(false) + expect( + await compute.updateServiceJobMetrics('none', expected(running), metrics) + ).to.equal(false) + }) + + it('lists the service jobs that hold a reservation', async () => { + await compute.newServiceJob(service({ serviceId: 'running' })) + await compute.newServiceJob( + service({ serviceId: 'error-unpaid', status: ServiceStatusNumber.Error }) + ) + await compute.newServiceJob( + service({ + serviceId: 'error-paid', + status: ServiceStatusNumber.Error, + payment: { claimTx: '0xclaim' } as any + }) + ) + await compute.newServiceJob( + service({ serviceId: 'expired', status: ServiceStatusNumber.Expired }) + ) + await compute.newServiceJob( + service({ serviceId: 'other-cluster', clusterHash: 'cluster-2' }) + ) + expect(serviceIds(await compute.getRunningServiceJobs('cluster-1'))).to.deep.equal([ + 'error-paid', + 'running' + ]) + expect(serviceIds(await compute.getRunningServiceJobs())).to.deep.equal([ + 'error-paid', + 'other-cluster', + 'running' + ]) + }) + + it('lists the service jobs to expire, to clean, and to start', async () => { + const past = Date.now() - HOUR_MS + await compute.newServiceJob(service({ serviceId: 'running-late', expiresAt: past })) + await compute.newServiceJob(service({ serviceId: 'running-in-time' })) + await compute.newServiceJob( + service({ + serviceId: 'expired-with-archive', + status: ServiceStatusNumber.Expired, + expiresAt: past, + outputArchives: [{ id: 'a' }] as any + }) + ) + await compute.newServiceJob( + service({ + serviceId: 'expired-clean', + status: ServiceStatusNumber.Expired, + expiresAt: past + }) + ) + await compute.newServiceJob( + service({ serviceId: 'starting', status: ServiceStatusNumber.Starting }) + ) + expect(serviceIds(await compute.getExpiredServiceJobs())).to.deep.equal([ + 'running-late' + ]) + expect(await compute.getExpiredServiceJobs('cluster-2')).to.deep.equal([]) + expect( + serviceIds(await compute.getExpiredServiceJobsBefore(Date.now())) + ).to.deep.equal(['expired-with-archive']) + expect(await compute.getExpiredServiceJobsBefore(past - HOUR_MS)).to.deep.equal([]) + expect(serviceIds(await compute.getPendingServiceStarts())).to.deep.equal([ + 'starting' + ]) + }) + + it('an update that removes the archives removes the job from the clean list', async () => { + const past = Date.now() - HOUR_MS + const expired = service({ + status: ServiceStatusNumber.Expired, + expiresAt: past, + outputArchives: [{ id: 'a' }] as any + }) + await compute.newServiceJob(expired) + await compute.updateServiceJob({ ...expired, outputArchives: [] }) + expect(await compute.getExpiredServiceJobsBefore(Date.now())).to.deep.equal([]) + }) + }) + + describe('service locks', () => { + const STALE_MS = 60_000 + + it('gives the lock to one holder only', async () => { + expect(await compute.acquireServiceLock('svc-1', 'a', STALE_MS)).to.equal(true) + expect(await compute.acquireServiceLock('svc-1', 'b', STALE_MS)).to.equal(false) + expect(await compute.isServiceLocked('svc-1', STALE_MS)).to.equal(true) + expect(await compute.isServiceLocked('svc-2', STALE_MS)).to.equal(false) + }) + + it('gives the lock to one of the holders that ask at the same time', async () => { + const holders = Array.from({ length: 10 }, (_, i) => `holder-${i}`) + const results = await Promise.all( + holders.map((holder) => compute.acquireServiceLock('svc-1', holder, STALE_MS)) + ) + expect(results.filter(Boolean)).to.have.length(1) + }) + + it('releases the lock of its holder only', async () => { + await compute.acquireServiceLock('svc-1', 'a', STALE_MS) + await compute.releaseServiceLock('svc-1', 'b') + expect(await compute.isServiceLocked('svc-1', STALE_MS)).to.equal(true) + await compute.releaseServiceLock('svc-1', 'a') + expect(await compute.acquireServiceLock('svc-1', 'b', STALE_MS)).to.equal(true) + }) + + it('gives a stale lock to a new holder, and a refresh keeps a lock fresh', async () => { + await compute.acquireServiceLock('svc-1', 'a', STALE_MS) + await db.run('UPDATE service_locks SET "acquiredAt" = ?', [ + Date.now() - 2 * STALE_MS + ]) + expect(await compute.isServiceLocked('svc-1', STALE_MS)).to.equal(false) + await compute.refreshServiceLocks('a') + expect(await compute.acquireServiceLock('svc-1', 'b', STALE_MS)).to.equal(false) + await db.run('UPDATE service_locks SET "acquiredAt" = ?', [ + Date.now() - 2 * STALE_MS + ]) + expect(await compute.acquireServiceLock('svc-1', 'b', STALE_MS)).to.equal(true) + }) + }) +}) diff --git a/src/test/unit/escrowFilters.test.ts b/src/test/unit/escrowFilters.test.ts index f07d7a18a..04b0b6f42 100644 --- a/src/test/unit/escrowFilters.test.ts +++ b/src/test/unit/escrowFilters.test.ts @@ -4,7 +4,6 @@ import { validateEscrowRanges, splitRanges, passesClientRanges, - buildTypesenseFilterBy, buildElasticQuery, NATIVE_RANGE_FIELDS, CLIENT_RANGE_FIELDS @@ -195,42 +194,6 @@ describe('escrow search filter helpers', () => { }) }) - describe('buildTypesenseFilterBy', () => { - it('builds exact-match clauses and backtick-escapes strings', () => { - const out = buildTypesenseFilterBy({ chainId: 8996, payer: '0xABC' }) - expect(out).to.contain('chainId:=8996') - expect(out).to.contain('payer:=`0xABC`') - expect(out).to.contain(' && ') - }) - - it('strips backticks from string values', () => { - const out = buildTypesenseFilterBy({ token: 'a`b' }) - expect(out).to.equal('token:=`ab`') - }) - - it('emits range operators for range objects', () => { - const out = buildTypesenseFilterBy({ block: { min: 100, max: 200 } }) - expect(out).to.equal('block:>=100 && block:<=200') - expect(buildTypesenseFilterBy({ block: { gt: 1, lt: 9 } })).to.equal( - 'block:>1 && block:<9' - ) - }) - - it('handles booleans and skips empty values', () => { - const out = buildTypesenseFilterBy({ - reclaimable: true, - payee: undefined, - token: '' - }) - expect(out).to.equal('reclaimable:=true') - }) - - it('ANDs an equality and a range on the same field (array value)', () => { - const out = buildTypesenseFilterBy({ chainId: [8996, { min: 1, max: 10 }] }) - expect(out).to.equal('chainId:=8996 && chainId:>=1 && chainId:<=10') - }) - }) - describe('buildElasticQuery', () => { it('builds term clauses for equality', () => { const q = buildElasticQuery({ chainId: 8996, payer: '0xabc' }) diff --git a/src/test/unit/fileInfoHandler.test.ts b/src/test/unit/fileInfoHandler.test.ts index 535e2fce1..ed78d388a 100644 --- a/src/test/unit/fileInfoHandler.test.ts +++ b/src/test/unit/fileInfoHandler.test.ts @@ -56,9 +56,7 @@ describe('FileInfoHandler ON-07 (header allow-list + rate limit; raw-file needs before(async () => { const config = await getConfiguration() const keyManager = new KeyManager(config) - // This suite only needs the SQLite-backed nonce/auth DBs. Drop the metadata DB url so - // Database.init() cannot depend on an Elasticsearch/Typesense being up. - const db = await Database.init({ ...config.dbConfig, url: '' }) + const db = await Database.init(config.dbConfig) node = OceanNode.getInstance(config, db, null, null, null, keyManager, null, true) }) @@ -144,7 +142,7 @@ describe('FileInfoHandler ON-07 (header allow-list + rate limit; raw-file needs range: 'bytes=0-10', authorization: 'Bearer secret-token', cookie: 'session=abc', - 'x-typesense-api-key': 'super-secret' + 'x-api-key': 'super-secret' } } as any, { @@ -160,7 +158,7 @@ describe('FileInfoHandler ON-07 (header allow-list + rate limit; raw-file needs // everything else dropped expect(received.authorization).to.equal(undefined) expect(received.cookie).to.equal(undefined) - expect(received['x-typesense-api-key']).to.equal(undefined) + expect(received['x-api-key']).to.equal(undefined) } finally { await stopServer(server) } diff --git a/src/test/unit/indexer/validation.test.ts b/src/test/unit/indexer/validation.test.ts index 1e1fd9d5d..afa33d5f1 100644 --- a/src/test/unit/indexer/validation.test.ts +++ b/src/test/unit/indexer/validation.test.ts @@ -57,7 +57,7 @@ describe('Schema validation tests', () => { nonce: {}, c2d: {}, authToken: {}, - sqliteConfig: {}, + config: {}, ddo: {}, indexer: {}, logs: {}, diff --git a/src/test/unit/maintenanceCoverage.test.ts b/src/test/unit/maintenanceCoverage.test.ts index 4ff563dd6..556b81e3d 100644 --- a/src/test/unit/maintenanceCoverage.test.ts +++ b/src/test/unit/maintenanceCoverage.test.ts @@ -227,7 +227,7 @@ describe('Maintenance mode - QA coverage gaps', () => { ) envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) config = await getConfiguration(true) - db = await Database.init({ url: 'http://localhost:9999', dbType: 'none' } as any) + db = await Database.init(config.dbConfig) keyManager = new KeyManager(config) blockchainRegistry = new BlockchainRegistry(keyManager, config) oceanNode = buildNode() @@ -425,7 +425,7 @@ describe('Maintenance mode - QA coverage gaps', () => { it('flag absent from DB -> loads OFF (fail-open)', async () => { await oceanNode.setMaintenanceMode(true) - sinon.stub(db.sqliteConfig, 'retrieveValue').resolves({ value: null } as any) + sinon.stub(db.config, 'retrieveValue').resolves({ value: null } as any) await oceanNode.loadMaintenanceMode() expect(oceanNode.isInMaintenance()).to.equal(false) }) @@ -434,14 +434,14 @@ describe('Maintenance mode - QA coverage gaps', () => { await oceanNode.setMaintenanceMode(false) // retrieveValue returns null (not { value: null }) on a DB error -- distinct from a // confirmed-absent key. A failed read must NOT silently resume accepting work. - sinon.stub(db.sqliteConfig, 'retrieveValue').resolves(null as any) + sinon.stub(db.config, 'retrieveValue').resolves(null as any) await oceanNode.loadMaintenanceMode() expect(oceanNode.isInMaintenance()).to.equal(true) }) - it('no sqliteConfig: toggle works in-memory only and does not throw', async () => { - const real = db.sqliteConfig - ;(db as any).sqliteConfig = undefined + it('no config DB: toggle works in-memory only and does not throw', async () => { + const real = db.config + ;(db as any).config = undefined try { await oceanNode.setMaintenanceMode(true) expect(oceanNode.isInMaintenance()).to.equal(true) @@ -449,7 +449,7 @@ describe('Maintenance mode - QA coverage gaps', () => { await oceanNode.loadMaintenanceMode() expect(oceanNode.isInMaintenance()).to.equal(true) } finally { - ;(db as any).sqliteConfig = real + ;(db as any).config = real } }) @@ -461,13 +461,13 @@ describe('Maintenance mode - QA coverage gaps', () => { oceanNode.setMaintenanceMode(false), oceanNode.setMaintenanceMode(true) ]) - const stored = await db.sqliteConfig.retrieveValue(MAINTENANCE_MODE_CONFIG_KEY) + const stored = await db.config.retrieveValue(MAINTENANCE_MODE_CONFIG_KEY) expect(oceanNode.isInMaintenance()).to.equal(stored.value === 'true') expect(oceanNode.isInMaintenance()).to.equal(true) // last call wins }) it('a failed write does not poison the lock chain (next toggle still works)', async () => { - const stub = sinon.stub(db.sqliteConfig, 'createOrUpdateConfig').resolves(null) + const stub = sinon.stub(db.config, 'createOrUpdateConfig').resolves(null) let threw = false try { await oceanNode.setMaintenanceMode(true) @@ -539,15 +539,15 @@ describe('Maintenance mode - QA coverage gaps', () => { expect(oceanNode.isInMaintenance()).to.equal(true) }) - it('env true with no sqliteConfig still boots into drain (process-only)', async () => { - const real = db.sqliteConfig - ;(db as any).sqliteConfig = undefined + it('env true with no config DB still boots into drain (process-only)', async () => { + const real = db.config + ;(db as any).config = undefined try { setEnv('true') await oceanNode.loadMaintenanceMode() expect(oceanNode.isInMaintenance()).to.equal(true) } finally { - ;(db as any).sqliteConfig = real + ;(db as any).config = real } }) diff --git a/src/test/unit/maintenanceMode.test.ts b/src/test/unit/maintenanceMode.test.ts index 97d3d5a65..6c6af669a 100644 --- a/src/test/unit/maintenanceMode.test.ts +++ b/src/test/unit/maintenanceMode.test.ts @@ -72,9 +72,7 @@ describe('Maintenance mode', () => { // Make sure no stale env leaks the flag into loadMaintenanceMode delete process.env.MAINTENANCE_MODE const config = await getConfiguration(true) - // SQLite-core-only DB: an invalid dbType skips the metadata DB (no Typesense/ES needed) - // while nonce/config/c2d/auth SQLite databases are always created. - db = await Database.init({ url: 'http://localhost:9999', dbType: 'none' } as any) + db = await Database.init(config.dbConfig) const keyManager = new KeyManager(config) const blockchainRegistry = new BlockchainRegistry(keyManager, config) oceanNode = OceanNode.getInstance( @@ -104,7 +102,7 @@ describe('Maintenance mode', () => { await oceanNode.setMaintenanceMode(true) expect(oceanNode.isInMaintenance()).to.equal(true) - const stored = await db.sqliteConfig.retrieveValue(MAINTENANCE_MODE_CONFIG_KEY) + const stored = await db.config.retrieveValue(MAINTENANCE_MODE_CONFIG_KEY) expect(stored.value).to.equal('true') // Simulate a restart: wipe the in-memory cache then reload from the persisted DB value. @@ -114,7 +112,7 @@ describe('Maintenance mode', () => { await oceanNode.setMaintenanceMode(false) expect(oceanNode.isInMaintenance()).to.equal(false) - const stored2 = await db.sqliteConfig.retrieveValue(MAINTENANCE_MODE_CONFIG_KEY) + const stored2 = await db.config.retrieveValue(MAINTENANCE_MODE_CONFIG_KEY) expect(stored2.value).to.equal('false') }) @@ -172,14 +170,14 @@ describe('Maintenance mode', () => { expect(resp.status.httpStatus).to.equal(200) expect(await streamBody(resp)).to.deep.equal({ maintenance: true }) expect(oceanNode.isInMaintenance()).to.equal(true) - const stored = await db.sqliteConfig.retrieveValue(MAINTENANCE_MODE_CONFIG_KEY) + const stored = await db.config.retrieveValue(MAINTENANCE_MODE_CONFIG_KEY) expect(stored.value).to.equal('true') await oceanNode.setMaintenanceMode(false) }) it('does NOT update the cache (and the handler returns 500) when the DB write fails', async () => { await oceanNode.setMaintenanceMode(false) - const stub = sinon.stub(db.sqliteConfig, 'createOrUpdateConfig').resolves(null) + const stub = sinon.stub(db.config, 'createOrUpdateConfig').resolves(null) try { let threw = false try { diff --git a/src/test/unit/nodeMetrics.test.ts b/src/test/unit/nodeMetrics.test.ts new file mode 100644 index 000000000..a2eaa32fe --- /dev/null +++ b/src/test/unit/nodeMetrics.test.ts @@ -0,0 +1,345 @@ +import { baseSnapshot, describeOnElasticsearch } from '../utils/utils.js' +import { expect } from 'chai' +import { Client } from '@elastic/elasticsearch' +import { floorToHour } from '../../components/database/nodeMetricsAggregate.js' +import { ElasticsearchNodeMetrics } from '../../components/database/ElasticsearchNodeMetrics.js' +import { + createElasticsearchClientWithRetry, + searchAllPages +} from '../../components/database/ElasticsearchConfigHelper.js' +import { getConfiguration } from '../../utils/index.js' + +const INSTANCE = 'test-instance' +const OTHER_INSTANCE = 'other-instance' + +describe('nodeMetricsAggregate.floorToHour', () => { + it('floors to the UTC hour boundary', () => { + const t = Date.UTC(2026, 6, 28, 10, 37, 42, 123) // 2026-07-28T10:37:42.123Z + const floored = floorToHour(t) + expect(floored).to.equal(Date.UTC(2026, 6, 28, 10, 0, 0, 0)) + }) + + it('is idempotent on an already-floored hour', () => { + const t = Date.UTC(2026, 6, 28, 10, 0, 0, 0) + expect(floorToHour(t)).to.equal(t) + }) +}) + +describeOnElasticsearch('ElasticsearchNodeMetrics', () => { + let db: ElasticsearchNodeMetrics + let client: Client + + before(async () => { + const { dbConfig } = await getConfiguration() + db = await ElasticsearchNodeMetrics.create(dbConfig) + client = await createElasticsearchClientWithRetry(dbConfig) + }) + + // sample inserts use refresh: 'wait_for', so no index refresh is needed first + beforeEach(async () => { + await client.deleteByQuery({ + index: ['node_metrics_samples', 'node_metrics_hourly'], + query: { match_all: {} }, + refresh: true, + conflicts: 'proceed' + }) + }) + + it('rollupHour averages scalars and records sampleCount', async () => { + const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) + await db.insertSample( + INSTANCE, + baseSnapshot({ + collectedAt: hour + 60_000, + cpu: { ...baseSnapshot().cpu, usagePercent: 10 } + }) + ) + await db.insertSample( + INSTANCE, + baseSnapshot({ + collectedAt: hour + 120_000, + cpu: { ...baseSnapshot().cpu, usagePercent: 30 } + }) + ) + const rolled = await db.rollupHour(INSTANCE, hour) + expect(rolled).to.equal(true) + + const rows = await db.getHourly(INSTANCE, hour, hour, 10) + expect(rows).to.have.length(1) + expect(rows[0].hourStart).to.equal(hour) + expect(rows[0].sampleCount).to.equal(2) + expect(rows[0].cpu.usagePercent).to.equal(20) // mean(10, 30) + }) + + it('rollupHour on an empty hour returns false and writes no row', async () => { + const hour = Date.UTC(2026, 6, 28, 11, 0, 0, 0) + const rolled = await db.rollupHour(INSTANCE, hour) + expect(rolled).to.equal(false) + expect(await db.getHourly(INSTANCE, hour, hour, 10)).to.have.length(0) + }) + + it('rollupHour only consumes samples within [hourStart, hourStart+1h) — partial-hour boundary', async () => { + const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) + const nextHour = hour + 60 * 60 * 1000 + await db.insertSample(INSTANCE, baseSnapshot({ collectedAt: hour + 1000 })) // inside hour + await db.insertSample(INSTANCE, baseSnapshot({ collectedAt: nextHour + 1000 })) // next hour, must not roll in + await db.rollupHour(INSTANCE, hour) + // the sample belonging to the next hour must still be present (not consumed) + const pending = await db.getPendingSampleHours(INSTANCE, nextHour + 60 * 60 * 1000) + expect(pending).to.include(nextHour) + expect(pending).to.not.include(hour) + }) + + it('rollupHour deletes the consumed raw samples after averaging', async () => { + const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) + await db.insertSample(INSTANCE, baseSnapshot({ collectedAt: hour + 1000 })) + await db.rollupHour(INSTANCE, hour) + // nothing left in that hour to roll up again (idempotent-ish: no samples, so returns false) + expect(await db.rollupHour(INSTANCE, hour)).to.equal(false) + }) + + it('a second roll-up keeps the first row and deletes the leftover samples', async () => { + const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) + for (const [offset, usagePercent] of [ + [1000, 10], + [2000, 30] + ]) { + await db.insertSample( + INSTANCE, + baseSnapshot({ + collectedAt: hour + offset, + cpu: { ...baseSnapshot().cpu, usagePercent } + }) + ) + } + const [counted] = await searchAllPages(client, { + index: 'node_metrics_samples', + query: { match_all: {} } + }) + expect(await db.rollupHour(INSTANCE, hour)).to.equal(true) + // the delete of the roll-up stopped part way, and a late sample came for the hour + await client.index({ + index: 'node_metrics_samples', + id: counted._id, + document: counted._source, + refresh: true + }) + await db.insertSample(INSTANCE, baseSnapshot({ collectedAt: hour + 3000 })) + expect(await db.rollupHour(INSTANCE, hour)).to.equal(false) + const rows = await db.getHourly(INSTANCE, hour, hour, 10) + expect(rows.map((row) => [row.sampleCount, row.cpu.usagePercent])).to.deep.equal([ + [2, 20] + ]) + expect(await db.getPartialHour(INSTANCE, hour)).to.equal(null) + }) + + it('a second instance sees none of the first instance samples or hourly rows', async () => { + const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) + const nextHour = hour + 60 * 60 * 1000 + await db.insertSample(INSTANCE, baseSnapshot({ collectedAt: hour + 1000 })) + await db.insertSample(INSTANCE, baseSnapshot({ collectedAt: nextHour + 1000 })) + await db.rollupHour(INSTANCE, hour) + + expect(await db.getPendingSampleHours(OTHER_INSTANCE, nextHour * 2)).to.deep.equal([]) + expect(await db.getPartialHour(OTHER_INSTANCE, nextHour)).to.equal(null) + expect(await db.getHourly(OTHER_INSTANCE, 0, nextHour, 10)).to.deep.equal([]) + expect(await db.rollupHour(OTHER_INSTANCE, nextHour)).to.equal(false) + expect(await db.purgeOldSamples(OTHER_INSTANCE, nextHour * 2)).to.equal(0) + expect(await db.deleteHourlyOlderThan(OTHER_INSTANCE, nextHour * 2)).to.equal(0) + + expect(await db.getHourly(INSTANCE, 0, nextHour, 10)).to.have.length(1) + expect(await db.getPendingSampleHours(INSTANCE, nextHour * 2)).to.deep.equal([ + nextHour + ]) + }) + + it('averages GPU metrics per resourceId and env resources per env+resource', async () => { + const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) + await db.insertSample( + INSTANCE, + baseSnapshot({ + collectedAt: hour + 1000, + gpu: [ + { + resourceId: 'gpu0', + vendor: 'nvidia', + utilizationPercent: 20, + memoryUsedBytes: 100 + } + ], + env: [{ env: 'envA', resource: 'cpu', total: 4, inUse: 2 }] + }) + ) + await db.insertSample( + INSTANCE, + baseSnapshot({ + collectedAt: hour + 2000, + gpu: [ + { + resourceId: 'gpu0', + vendor: 'nvidia', + utilizationPercent: 80, + memoryUsedBytes: 300 + } + ], + env: [{ env: 'envA', resource: 'cpu', total: 4, inUse: 4 }] + }) + ) + await db.rollupHour(INSTANCE, hour) + const [row] = await db.getHourly(INSTANCE, hour, hour, 10) + expect(row.gpu).to.have.length(1) + expect(row.gpu[0].resourceId).to.equal('gpu0') + expect(row.gpu[0].utilizationPercent).to.equal(50) // mean(20, 80) + expect(row.gpu[0].memoryUsedBytes).to.equal(200) // mean(100, 300) + expect(row.env).to.have.length(1) + expect(row.env[0].inUse).to.equal(3) // mean(2, 4) + }) + + it('GPUs present in only some samples do not have their mean dragged down by absent samples', async () => { + const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) + // gpu0 reports temperatureC only on the second sample + await db.insertSample( + INSTANCE, + baseSnapshot({ + collectedAt: hour + 1000, + gpu: [{ resourceId: 'gpu0', utilizationPercent: 10 }] + }) + ) + await db.insertSample( + INSTANCE, + baseSnapshot({ + collectedAt: hour + 2000, + gpu: [{ resourceId: 'gpu0', utilizationPercent: 30, temperatureC: 60 }] + }) + ) + await db.rollupHour(INSTANCE, hour) + const [row] = await db.getHourly(INSTANCE, hour, hour, 10) + expect(row.gpu[0].utilizationPercent).to.equal(20) // mean over 2 samples + expect(row.gpu[0].temperatureC).to.equal(60) // mean over 1 sample that reported it, not 30 + }) + + it('searchAllPages reads every match across pages, in order', async () => { + const start = Date.now() - 5 * 60 * 1000 + for (let i = 0; i < 5; i++) { + await db.insertSample( + INSTANCE, + baseSnapshot({ collectedAt: start + i * 60 * 1000 }) + ) + } + // page size 2 -> three pages for five samples + const hits = await searchAllPages<{ collectedAt: number }>( + client, + { + index: 'node_metrics_samples', + query: { term: { instanceId: INSTANCE } }, + sort: [{ collectedAt: { order: 'asc' } }], + _source: ['collectedAt'] + }, + 2 + ) + expect(hits.map((hit) => hit._source.collectedAt)).to.deep.equal( + [0, 1, 2, 3, 4].map((i) => start + i * 60 * 1000) + ) + }) + + it('purgeOldSamples deletes samples older than the cutoff and returns the count', async () => { + const now = Date.now() + await db.insertSample( + INSTANCE, + baseSnapshot({ collectedAt: now - 4 * 60 * 60 * 1000 }) + ) // old + await db.insertSample(INSTANCE, baseSnapshot({ collectedAt: now })) // recent + const purged = await db.purgeOldSamples(INSTANCE, now - 3 * 60 * 60 * 1000) + expect(purged).to.equal(1) + }) + + it('deleteHourlyOlderThan enforces retention and returns the count deleted', async () => { + const oldHour = Date.UTC(2020, 0, 1) + const recentHour = floorToHour(Date.now()) + await db.insertSample(INSTANCE, baseSnapshot({ collectedAt: oldHour + 1000 })) + await db.rollupHour(INSTANCE, oldHour) + await db.insertSample(INSTANCE, baseSnapshot({ collectedAt: recentHour + 1000 })) + await db.rollupHour(INSTANCE, recentHour) + + const deleted = await db.deleteHourlyOlderThan( + INSTANCE, + Date.now() - 180 * 24 * 60 * 60 * 1000 + ) + expect(deleted).to.equal(1) + const remaining = await db.getHourly(INSTANCE, 0, Date.now(), 1000) + expect(remaining).to.have.length(1) + expect(remaining[0].hourStart).to.equal(recentHour) + }) + + it('getHourly returns ordered buckets within range, capped at limit', async () => { + const h0 = Date.UTC(2026, 6, 28, 8, 0, 0, 0) + const h1 = h0 + 60 * 60 * 1000 + const h2 = h1 + 60 * 60 * 1000 + for (const h of [h0, h1, h2]) { + await db.insertSample(INSTANCE, baseSnapshot({ collectedAt: h + 1000 })) + await db.rollupHour(INSTANCE, h) + } + const rows = await db.getHourly(INSTANCE, h0, h2, 2) + expect(rows).to.have.length(2) + expect(rows[0].hourStart).to.equal(h0) + expect(rows[1].hourStart).to.equal(h1) + }) + + it('JSON round-trips gpu/env through storage (safeParse recovers structured fields)', async () => { + const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) + await db.insertSample( + INSTANCE, + baseSnapshot({ + collectedAt: hour + 1000, + gpu: [{ resourceId: 'gpu0', vendor: 'nvidia', powerWatts: 88 }], + env: [{ env: 'envA', resource: 'gpu0', total: 1, inUse: 1 }] + }) + ) + await db.rollupHour(INSTANCE, hour) + const [row] = await db.getHourly(INSTANCE, hour, hour, 10) + expect(row.gpu[0]).to.include({ + resourceId: 'gpu0', + vendor: 'nvidia', + powerWatts: 88 + }) + expect(row.env[0]).to.deep.equal({ + env: 'envA', + resource: 'gpu0', + total: 1, + inUse: 1 + }) + }) + + it('getPartialHour averages the raw (un-rolled) samples and flags partial:true', async () => { + const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) + await db.insertSample( + INSTANCE, + baseSnapshot({ + collectedAt: hour + 60_000, + cpu: { ...baseSnapshot().cpu, usagePercent: 10 } + }) + ) + await db.insertSample( + INSTANCE, + baseSnapshot({ + collectedAt: hour + 120_000, + cpu: { ...baseSnapshot().cpu, usagePercent: 30 } + }) + ) + const partial = await db.getPartialHour(INSTANCE, hour) + expect(partial).to.not.equal(null) + expect(partial!.partial).to.equal(true) + expect(partial!.hourStart).to.equal(hour) + expect(partial!.sampleCount).to.equal(2) + expect(partial!.cpu.usagePercent).to.equal(20) // mean(10, 30) — same averaging as rollupHour + }) + + it('getPartialHour returns null when the hour has no samples, and does not consume samples', async () => { + const hour = Date.UTC(2026, 6, 28, 12, 0, 0, 0) + expect(await db.getPartialHour(INSTANCE, hour)).to.equal(null) + // it is read-only: a sample in a populated hour survives a getPartialHour call + await db.insertSample(INSTANCE, baseSnapshot({ collectedAt: hour + 1000 })) + expect((await db.getPartialHour(INSTANCE, hour))!.sampleCount).to.equal(1) + expect((await db.getPartialHour(INSTANCE, hour))!.sampleCount).to.equal(1) + }) +}) diff --git a/src/test/unit/nodeMetricsHandlers.test.ts b/src/test/unit/nodeMetricsHandlers.test.ts index 0c88e48ee..e668b416b 100644 --- a/src/test/unit/nodeMetricsHandlers.test.ts +++ b/src/test/unit/nodeMetricsHandlers.test.ts @@ -1,3 +1,4 @@ +import { baseSnapshot, describeOnElasticsearch } from '../utils/utils.js' import { expect } from 'chai' import { Readable } from 'stream' import { @@ -9,17 +10,13 @@ import { GetNodeMetricsCommand, GetNodeMetricsHistoryCommand } from '../../@types/commands.js' -import { - NodeMetricsDatabase, - floorToHour -} from '../../components/database/sqliteNodeMetrics.js' -import type { NodeMetricsSnapshot } from '../../@types/nodeMetrics.js' +import { Client } from '@elastic/elasticsearch' +import { floorToHour } from '../../components/database/nodeMetricsAggregate.js' +import { ElasticsearchNodeMetrics } from '../../components/database/ElasticsearchNodeMetrics.js' +import { createElasticsearchClientWithRetry } from '../../components/database/ElasticsearchConfigHelper.js' import { getConfiguration } from '../../utils/config.js' import { OceanNode } from '../../OceanNode.js' import { KeyManager } from '../../components/KeyManager/index.js' -import fs from 'fs' -import path from 'path' -import os from 'os' import { getMetricsRetentionDays, isNodeMetricsHistoryEnabled @@ -31,61 +28,46 @@ async function streamToJson(stream: Readable) { return JSON.parse(Buffer.concat(chunks).toString('utf-8')) } -function baseSnapshot(overrides: Partial = {}): NodeMetricsSnapshot { - return { - collectedAt: Date.now(), - hasAggregate: true, - cpu: { - usagePercent: 10, - coresAllocated: 2, - hostCores: 8, - throttledCount: 0, - loadAverage: [] - }, - memory: { - usedBytes: 100, - limitBytes: 1000, - hostFreeBytes: 500, - hostTotalBytes: 2000 - }, - disk: { usedBytes: 300 }, - network: { rxBytes: 10, txBytes: 20 }, - jobs: { running: 1, runningFree: 0, queued: 0, queuedFree: 0 }, - gpu: [], - env: [], - meta: { sampledContainers: 1, oldestSampleAgeSeconds: 5 }, - ...overrides - } -} - // Spies on getHourly() so param clamping/capping can be asserted without needing thousands of // real rows in the DB. -class SpyingNodeMetricsDatabase extends NodeMetricsDatabase { - public lastCall: { from: number; to: number; limit: number } | undefined - getHourly(from: number, to: number, limit: number) { - this.lastCall = { from, to, limit } - return super.getHourly(from, to, limit) +interface SpyingNodeMetricsDatabase extends ElasticsearchNodeMetrics { + lastCall?: { instanceId: string; from: number; to: number; limit: number } +} + +function spyOnGetHourly(store: ElasticsearchNodeMetrics): SpyingNodeMetricsDatabase { + const spy: SpyingNodeMetricsDatabase = Object.create(store) + spy.getHourly = (instanceId: string, from: number, to: number, limit: number) => { + spy.lastCall = { instanceId, from, to, limit } + return store.getHourly(instanceId, from, to, limit) } + return spy } -describe('nodeMetrics handlers', () => { +describeOnElasticsearch('nodeMetrics handlers', () => { let node: OceanNode - let tempDir: string + let instanceId: string + let store: ElasticsearchNodeMetrics + let client: Client let metricsDb: SpyingNodeMetricsDatabase before(async () => { const config = await getConfiguration() const keyManager = new KeyManager(config) node = OceanNode.getInstance(config, null, null, null, null, keyManager, null, true) + instanceId = node.getInstanceId() + store = await ElasticsearchNodeMetrics.create(config.dbConfig) + client = await createElasticsearchClientWithRetry(config.dbConfig) }) - beforeEach(() => { - tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'node-metrics-handler-test-')) - metricsDb = new SpyingNodeMetricsDatabase(path.join(tempDir, 'nodeMetrics.sqlite')) - }) - - afterEach(() => { - fs.rmSync(tempDir, { recursive: true, force: true }) + // sample inserts use refresh: 'wait_for', so no index refresh is needed first + beforeEach(async () => { + await client.deleteByQuery({ + index: ['node_metrics_samples', 'node_metrics_hourly'], + query: { match_all: {} }, + refresh: true, + conflicts: 'proceed' + }) + metricsDb = spyOnGetHourly(store) }) describe('GetNodeMetricsHandler (live snapshot)', () => { @@ -105,7 +87,7 @@ describe('nodeMetrics handlers', () => { describe('GetNodeMetricsHistoryHandler (param parse/validate/clamp/cap)', () => { function nodeWithMetricsDb(): OceanNode { // Override getDatabase() to hand back only what the handler reads (db.nodeMetrics), - // avoiding any dependency on a real metadata DB or the shared repo-level SQLite files. + // avoiding any dependency on the rest of the database. const fakeNode = Object.create(node) fakeNode.getDatabase = () => Promise.resolve({ nodeMetrics: metricsDb }) return fakeNode as OceanNode @@ -235,13 +217,14 @@ describe('nodeMetrics handlers', () => { } await new GetNodeMetricsHistoryHandler(nodeWithMetricsDb()).handle(command) expect(metricsDb.lastCall?.limit).to.equal(10000) + expect(metricsDb.lastCall?.instanceId).to.equal(instanceId) }) it('returns real hourly buckets stored in the database within the requested range', async () => { // Use a recent hour (a few hours before "now") so it stays inside the retention window. const hour = floorToHour(Date.now() - 5 * 60 * 60 * 1000) - metricsDb.insertSample(baseSnapshot({ collectedAt: hour + 1000 })) - metricsDb.rollupHour(hour) + await metricsDb.insertSample(instanceId, baseSnapshot({ collectedAt: hour + 1000 })) + await metricsDb.rollupHour(instanceId, hour) const command: GetNodeMetricsHistoryCommand = { command: PROTOCOL_COMMANDS.GET_NODE_METRICS_HISTORY, @@ -277,13 +260,15 @@ describe('nodeMetrics handlers', () => { it('appends a live partial:true bucket for the current, not-yet-rolled-up hour', async () => { // Raw samples in the CURRENT hour, deliberately NOT rolled up. const currentHour = floorToHour(Date.now()) - metricsDb.insertSample( + await metricsDb.insertSample( + instanceId, baseSnapshot({ collectedAt: currentHour + 1000, cpu: { ...baseSnapshot().cpu, usagePercent: 40 } }) ) - metricsDb.insertSample( + await metricsDb.insertSample( + instanceId, baseSnapshot({ collectedAt: currentHour + 2000, cpu: { ...baseSnapshot().cpu, usagePercent: 60 } @@ -303,8 +288,11 @@ describe('nodeMetrics handlers', () => { it('does not append a partial bucket when the current hour has already been rolled up', async () => { const currentHour = floorToHour(Date.now()) - metricsDb.insertSample(baseSnapshot({ collectedAt: currentHour + 1000 })) - metricsDb.rollupHour(currentHour) // stored → no partial should be added on top + await metricsDb.insertSample( + instanceId, + baseSnapshot({ collectedAt: currentHour + 1000 }) + ) + await metricsDb.rollupHour(instanceId, currentHour) // stored → no partial should be added on top const result = await new GetNodeMetricsHistoryHandler(nodeWithMetricsDb()).handle({ command: PROTOCOL_COMMANDS.GET_NODE_METRICS_HISTORY diff --git a/src/test/unit/oceanP2P.test.ts b/src/test/unit/oceanP2P.test.ts index 3c0cf82aa..81b3d1de9 100644 --- a/src/test/unit/oceanP2P.test.ts +++ b/src/test/unit/oceanP2P.test.ts @@ -107,7 +107,6 @@ describe('OceanP2P Test without DB_URL set', () => { // NO need to mess more with process.env variables const config = await getConfiguration(true) config.dbConfig.url = '' - config.dbConfig.dbType = null // assert(config.dbConfig.url === '', 'DB URL should not be set') const keyManager = new KeyManager(config) const p2pNode = new OceanP2P(config, keyManager) diff --git a/src/test/unit/p2p/provideLimiter.test.ts b/src/test/unit/p2p/provideLimiter.test.ts index 8e14dedca..155de0503 100644 --- a/src/test/unit/p2p/provideLimiter.test.ts +++ b/src/test/unit/p2p/provideLimiter.test.ts @@ -81,13 +81,8 @@ describe('Bulk DHT provides share one concurrency ceiling', () => { getDatabase: () => Promise.resolve({ ddo: { - search: () => - Promise.resolve([ - { - found: 12, - hits: ddoBatch(12, 'republish').map((ddo) => ({ document: ddo })) - } - ]) + find: () => + Promise.resolve({ results: ddoBatch(12, 'republish'), totalResults: 12 }) } }), getP2PNode: () => node diff --git a/src/test/unit/p2p/republishStoredDdos.test.ts b/src/test/unit/p2p/republishStoredDdos.test.ts index 4d5b665b5..82af834db 100644 --- a/src/test/unit/p2p/republishStoredDdos.test.ts +++ b/src/test/unit/p2p/republishStoredDdos.test.ts @@ -6,22 +6,8 @@ import { OceanNode } from '../../../OceanNode.js' /** * The startup re-advertise is the only thing that can put this node's provider records back * after its p2p datastore came up empty - kad-dht's reprovider works off that datastore, so - * with nothing in it there is nothing for it to refresh. It used to send `{ q: '*' }` and read - * `result[0].found` / `result[0].hits`, which is neither backend's contract: - * - * - Elasticsearch spreads the query object into the request body, where `q` is not a key. The - * request fails with `parsing_exception: Unknown key for a VALUE_STRING in [q]`, the failure - * is caught and skipped per schema, and the result comes back empty - so the job logged - * "There is nothing to republish" on a store holding hundreds of DDOs, and logged it as a - * success. - * - Typesense does accept `q: '*'`, but returns one response per collection and the first one - * is the deprecated-DDO collection, which on a healthy node is empty. `result[0].found` is - * therefore 0, so the same "nothing to republish" branch was taken, and even when it was not - * only `result[0].hits` - one page of one collection - would ever have been iterated. - * - * The fakes below reproduce each backend's paging contract and its answer shape rather than - * asserting on the query string, so a change that sends a form the backend rejects fails here - * the same way it fails in production. + * with nothing in it there is nothing for it to refresh. It walks every DDO of the store, of + * every version and state, one page at a time. */ const PAGE_SIZE = 100 // must match REPUBLISH_PAGE_SIZE in the job under test @@ -32,84 +18,32 @@ const docsNamed = (tag: string, count: number): any[] => version: '4.7.0' })) -interface RecordedQuery { - query: Record - perPage?: number - page?: number +interface RecordedFind { + conditions: any[] + page: { offset: number; limit: number } } /** - * Stands in for `ElasticsearchDdoDatabase`: schemas are named by `index`, the query object is - * the request body (so any key Elasticsearch does not know is fatal for that schema, caught and - * skipped), `from` is a 1-based page number that the database layer turns into an offset, and - * the answer is one bare array of documents per schema that had hits - schemas with none are - * absent entirely, and there is no total anywhere in the response. + * Stands in for the DDO store: `find` gives the DDOs of all versions in the order of the id, + * after the id of the range condition when there is one, one page at a time. */ -function elasticFake(collections: Array<[string, any[]]>): { - db: any - queries: RecordedQuery[] -} { - const queries: RecordedQuery[] = [] - const bodyKeys = ['query', 'size', 'from', 'sort', 'index'] +function storeFake(docs: any[]): { db: any; finds: RecordedFind[] } { + const finds: RecordedFind[] = [] + const byId = [...docs].sort((a, b) => (a.id < b.id ? -1 : 1)) const db = { - getSchemas: () => collections.map(([index]) => ({ index })), - search: (query: Record) => { - queries.push({ query }) - const size = query.size || PAGE_SIZE - const offset = (query.from || 1) * size - size - const results: any[] = [] - for (const [, docs] of collections) { - const rejected = Object.keys(query).some((key) => !bodyKeys.includes(key)) - if (rejected || !query.query?.match_all) { - // the per-schema catch that swallowed the `q` failure - continue - } - const page = docs.slice(offset, offset + size) - if (page.length > 0) { - results.push(page.map((doc) => ({ ...doc }))) - } - } - return Promise.resolve(results) - } - } - return { db, queries } -} - -/** - * Stands in for `TypesenseDdoDatabase`: schemas are named by `name`, paging is passed as the - * second and third arguments rather than inside the query, `q` is required and `'*'` is the - * match-everything wildcard, and the answer is one object per collection *always* - each with - * that collection's own `found` and a `hits[]` whose entries wrap the document. - */ -function typesenseFake(collections: Array<[string, any[]]>): { - db: any - queries: RecordedQuery[] -} { - const queries: RecordedQuery[] = [] - const db = { - getSchemas: () => collections.map(([name]) => ({ name })), - search: (query: Record, perPage?: number, page?: number) => { - queries.push({ query, perPage, page }) - if (query.q !== '*') { - // the real client answers a missing/!invalid `q` with a 400, which the database layer - // turns into a null after logging - return Promise.resolve(null) - } - const size = Math.min(perPage || 250, 250) - const pageNumber = page || 1 - return Promise.resolve( - collections.map(([, docs]) => ({ - found: docs.length, - out_of: docs.length, - page: pageNumber, - hits: docs - .slice((pageNumber - 1) * size, pageNumber * size) - .map((doc) => ({ document: { ...doc } })) - })) - ) + find: (conditions: any[], sort: any[], page: { offset: number; limit: number }) => { + finds.push({ conditions, page }) + const after = conditions[0]?.range?.gt + const results = byId + .filter((doc) => after === undefined || doc.id > after) + .slice(page.offset, page.offset + page.limit) + return Promise.resolve({ + results: results.map((doc) => ({ ...doc })), + totalResults: docs.length + }) } } - return { db, queries } + return { db, finds } } function nodeWith(ddoDb: any): { node: OceanNode; advertised: string[]; cached: any[] } { @@ -131,104 +65,46 @@ function nodeWith(ddoDb: any): { node: OceanNode; advertised: string[]; cached: } describe('The startup re-advertise republishes everything the node holds', () => { - it('finds documents on Elasticsearch, where the old query form failed outright', async () => { - const collections: Array<[string, any[]]> = [ - ['op_ddo_short', []], - ['op_ddo_v4.7.0', docsNamed('current', 150)], - ['op_ddo_v4.5.0', docsNamed('older', 80)] - ] - // what the pre-fix job sent and read, against an identical store - on its own fake so the - // recorded queries below contain only what the job itself issued - const oldShape: any = await elasticFake(collections).db.search({ q: '*' }) - expect( - !!(oldShape && oldShape.length > 0 && oldShape[0].found), - 'the old guard should be provably false here' - ).to.equal(false) - - const { db, queries } = elasticFake(collections) - + it('walks all DDOs, after the last id of each page, with no offset', async () => { + const docs = [...docsNamed('current', 150), ...docsNamed('older', 80)] + const { db, finds } = storeFake(docs) const { node, advertised, cached } = nodeWith(db) await p2pAnnounceDDOS(node) expect(new Set(advertised).size).to.equal(230) - expect(advertised.filter((did) => did.startsWith('did:op:current-')).length).to.equal( - 150 - ) - expect(advertised.filter((did) => did.startsWith('did:op:older-')).length).to.equal( - 80 - ) expect(cached.length).to.equal(230) - // no request may carry the URI-style `q` into the request body - expect(queries.every(({ query }) => query.q === undefined)).to.equal(true) - expect(queries.every(({ query }) => query.query?.match_all !== undefined)).to.equal( - true - ) - }) - - it('walks past the first page and past the first collection on Typesense', async () => { - // the first collection is empty, exactly as op_ddo_short is on a node holding current - // DDOs - which is what made `result[0].found` read as "nothing to republish" - const collections: Array<[string, any[]]> = [ - ['op_ddo_short', []], - ['op_ddo_v4.7.0', docsNamed('current', 150)], - ['op_ddo_v4.5.0', docsNamed('older', 80)] - ] - const oldShape: any = await typesenseFake(collections).db.search({ q: '*' }) - expect( - !!(oldShape && oldShape.length > 0 && oldShape[0].found), - 'the old guard should be provably false here' - ).to.equal(false) - - const { db, queries } = typesenseFake(collections) - - const { node, advertised } = nodeWith(db) - await p2pAnnounceDDOS(node) - - expect(new Set(advertised).size).to.equal(230) - expect(advertised.filter((did) => did.startsWith('did:op:current-')).length).to.equal( - 150 - ) - expect(advertised.filter((did) => did.startsWith('did:op:older-')).length).to.equal( - 80 - ) - // 150 documents in one collection cannot fit in a single page of 100 - expect(Math.max(...queries.map(({ page }) => page))).to.be.greaterThan(1) - expect(queries.every(({ query }) => query.q === '*')).to.equal(true) - expect(queries.every(({ perPage }) => perPage <= 250)).to.equal(true) + // every state and version: no condition other than the id of the page before + expect(finds[0].conditions).to.deep.equal([]) + expect(finds.slice(1).map(({ conditions }) => conditions[0].range.gt)).to.deep.equal([ + advertised[PAGE_SIZE - 1], + advertised[2 * PAGE_SIZE - 1] + ]) + expect(finds.every(({ page }) => page.offset === 0)).to.equal(true) }) it('stops when the last page is exactly full rather than looping', async () => { // a count that is an exact multiple of the page size is the case where "this page was not // full" cannot be the stop signal: the job has to fetch one more page and get nothing - for (const fake of [elasticFake, typesenseFake]) { - const { db, queries } = fake([['op_ddo_v4.7.0', docsNamed('exact', PAGE_SIZE * 2)]]) - const { node, advertised } = nodeWith(db) - await p2pAnnounceDDOS(node) - expect(new Set(advertised).size).to.equal(PAGE_SIZE * 2) - expect(queries.length).to.equal(3) - } + const { db, finds } = storeFake(docsNamed('exact', PAGE_SIZE * 2)) + const { node, advertised } = nodeWith(db) + await p2pAnnounceDDOS(node) + expect(new Set(advertised).size).to.equal(PAGE_SIZE * 2) + expect(finds.length).to.equal(3) }) it('makes one request and no more when everything fits in a page', async () => { - for (const fake of [elasticFake, typesenseFake]) { - const { db, queries } = fake([['op_ddo_v4.7.0', docsNamed('few', 7)]]) - const { node, advertised } = nodeWith(db) - await p2pAnnounceDDOS(node) - expect(advertised.length).to.equal(7) - expect(queries.length).to.equal(1) - } + const { db, finds } = storeFake(docsNamed('few', 7)) + const { node, advertised } = nodeWith(db) + await p2pAnnounceDDOS(node) + expect(advertised.length).to.equal(7) + expect(finds.length).to.equal(1) }) it('advertises nothing when the store really is empty', async () => { - for (const fake of [elasticFake, typesenseFake]) { - const { db } = fake([ - ['op_ddo_short', []], - ['op_ddo_v4.7.0', []] - ]) - const { node, advertised } = nodeWith(db) - await p2pAnnounceDDOS(node) - expect(advertised.length).to.equal(0) - } + const { db } = storeFake([]) + const { node, advertised } = nodeWith(db) + await p2pAnnounceDDOS(node) + expect(advertised.length).to.equal(0) }) }) @@ -251,10 +127,9 @@ describe('The re-advertise runs once per process, not on a timer', () => { let environmentFetches = 0 const ddoDb = { - getSchemas: (): any[] => [], - search: () => { + find: () => { ddoSearches++ - return Promise.resolve([]) + return Promise.resolve({ results: [], totalResults: 0 }) } } const node = { diff --git a/src/test/unit/persistentStorageSharing.test.ts b/src/test/unit/persistentStorageSharing.test.ts index 4b1abe441..7547904c5 100644 --- a/src/test/unit/persistentStorageSharing.test.ts +++ b/src/test/unit/persistentStorageSharing.test.ts @@ -1,15 +1,39 @@ +import { describeOnElasticsearch } from '../utils/utils.js' import { expect } from 'chai' import fsp from 'fs/promises' import os from 'os' import path from 'path' import { Wallet } from 'ethers' +import { Client } from '@elastic/elasticsearch' import type { OceanNode } from '../../OceanNode.js' import type { AccessList } from '../../@types/AccessList.js' import { PersistentStorageLocalFS } from '../../components/persistentStorage/PersistentStorageLocalFS.js' import { PersistentStorageAccessDeniedError } from '../../components/persistentStorage/PersistentStorageFactory.js' +import { ElasticsearchBucketRegistry } from '../../components/database/ElasticsearchBucketRegistry.js' +import { createElasticsearchClientWithRetry } from '../../components/database/ElasticsearchConfigHelper.js' +import { getConfiguration } from '../../utils/index.js' -describe('Persistent storage bucket sharing toggle', () => { +let registry: ElasticsearchBucketRegistry +let esClient: Client + +async function wipeBuckets(): Promise { + await esClient.deleteByQuery({ + index: 'buckets', + query: { match_all: {} }, + refresh: true, + conflicts: 'proceed' + }) +} + +async function initRegistry(): Promise { + if (registry) return + const { dbConfig } = await getConfiguration() + registry = await ElasticsearchBucketRegistry.create(dbConfig) + esClient = await createElasticsearchClientWithRetry(dbConfig) +} + +describeOnElasticsearch('Persistent storage bucket sharing toggle', () => { const owner = Wallet.createRandom().address const other = Wallet.createRandom().address const bucketAcl: AccessList[] = [{ '8996': [Wallet.createRandom().address] }] @@ -23,6 +47,9 @@ describe('Persistent storage bucket sharing toggle', () => { let privateBucketId: string before(async () => { + await initRegistry() + // buckets are created once for the whole suite, so wipe before, not beforeEach + await wipeBuckets() folder = await fsp.mkdtemp(path.join(os.tmpdir(), 'ocean-ps-sharing-')) const fakeNode = { getConfig: () => ({ @@ -35,7 +62,7 @@ describe('Persistent storage bucket sharing toggle', () => { } }) } as unknown as OceanNode - backend = new PersistentStorageLocalFS(fakeNode) + backend = new PersistentStorageLocalFS(fakeNode, registry) // Stand-in for the on-chain access list check: every address is on any non-empty ACL // (like the real check, an empty ACL lets nobody in). backend.isAllowed = (_consumer, accessLists) => { @@ -124,3 +151,56 @@ describe('Persistent storage bucket sharing toggle', () => { expect(aclChecks).to.equal(0) }) }) + +describeOnElasticsearch('Elasticsearch bucket registry', () => { + const owner = Wallet.createRandom().address + + before(initRegistry) + beforeEach(wipeBuckets) + + it('returns an upserted bucket and null for an unknown one', async () => { + await registry.upsertBucket('b1', owner, '[]', 100, 'first') + expect(await registry.getBucket('b1')).to.deep.equal({ + bucketId: 'b1', + owner, + accessListJson: '[]', + createdAt: 100, + label: 'first' + }) + expect(await registry.getBucket('missing')).to.equal(null) + }) + + it('re-upsert updates the access list only and keeps the old label', async () => { + await registry.upsertBucket('b1', owner, '[]', 100, 'first') + await registry.upsertBucket('b1', owner, '[{"1":["0xabc"]}]', 200, 'second') + const row = await registry.getBucket('b1') + expect(row.accessListJson).to.equal('[{"1":["0xabc"]}]') + expect(row.label).to.equal('first') + expect(row.createdAt).to.equal(100) + }) + + it('lists buckets by exact owner, oldest first', async () => { + await registry.upsertBucket('late', owner, '[]', 300, null) + await registry.upsertBucket('early', owner, '[]', 100, null) + await registry.upsertBucket('other', Wallet.createRandom().address, '[]', 200, null) + const rows = await registry.listBucketsByOwner(owner) + expect(rows.map((r) => r.bucketId)).to.deep.equal(['early', 'late']) + expect(await registry.listBucketsByOwner(owner.toLowerCase())).to.deep.equal([]) + }) + + it('deletes a bucket once', async () => { + await registry.upsertBucket('b1', owner, '[]', 100, null) + expect(await registry.deleteBucket('b1')).to.equal(true) + expect(await registry.deleteBucket('b1')).to.equal(false) + expect(await registry.getBucket('b1')).to.equal(null) + }) + + it('updates the label only for the owner', async () => { + await registry.upsertBucket('b1', owner, '[]', 100, 'first') + const stranger = Wallet.createRandom().address + expect(await registry.updateBucketLabel('b1', stranger, 'hacked')).to.equal(false) + expect(await registry.updateBucketLabel('missing', owner, 'x')).to.equal(false) + expect(await registry.updateBucketLabel('b1', owner, null)).to.equal(true) + expect((await registry.getBucket('b1')).label).to.equal(null) + }) +}) diff --git a/src/test/unit/queryHandler.test.ts b/src/test/unit/queryHandler.test.ts new file mode 100644 index 000000000..a9aee931f --- /dev/null +++ b/src/test/unit/queryHandler.test.ts @@ -0,0 +1,180 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import { Readable } from 'stream' +import { + QueryDdoStateHandler, + QueryHandler +} from '../../components/core/handler/queryHandler.js' +import { ElasticsearchDdoDatabase } from '../../components/database/ElasticSearchDatabase.js' +import { PROTOCOL_COMMANDS } from '../../utils/constants.js' +import { streamToString } from '../../utils/util.js' + +const SCHEMAS = [{ index: 'op_ddo_v4.1.0' }, { index: 'op_ddo_v5.0.0' }] +const DDO = { id: 'did:op:1' } + +describe('Query handler', () => { + let ddoFind: sinon.SinonStub + let ddoTags: sinon.SinonStub + let orderFind: sinon.SinonStub + let ddoStateFind: sinon.SinonStub + let node: any + + beforeEach(() => { + ddoFind = sinon.stub().resolves({ results: [DDO], totalResults: 7 }) + ddoTags = sinon.stub().resolves([{ tag: 'weather', count: 2 }]) + orderFind = sinon.stub().resolves({ results: [{ id: '0xorder' }], totalResults: 1 }) + ddoStateFind = sinon.stub().resolves([{ id: 'did:op:1' }]) + const database = { + ddo: { find: ddoFind, tags: ddoTags, getSchemas: () => SCHEMAS }, + order: { find: orderFind }, + ddoState: { find: ddoStateFind } + } + node = { + getRequestMap: () => new Map(), + getConfig: () => ({ rateLimit: 1000 }), + getDatabase: () => Promise.resolve(database) + } + }) + + const run = async (query: unknown) => { + const response = await new QueryHandler(node).handle({ + command: PROTOCOL_COMMANDS.QUERY, + query, + caller: '127.0.0.1' + }) + const body = response.stream + ? JSON.parse(await streamToString(response.stream as Readable)) + : undefined + return { status: response.status, body } + } + + it('searches the assets: conditions, sort and page; answers results and total', async () => { + const { status, body } = await run({ + filter: { chainIds: [8996], types: ['dataset'] }, + sort: { by: 'orders', order: 'asc' }, + page: 2, + size: 5 + }) + expect(status.httpStatus).to.equal(200) + expect(body).to.deep.equal({ results: [DDO], totalResults: 7 }) + const [conditions, sort, page] = ddoFind.firstCall.args + expect(conditions).to.deep.include({ + paths: ['chainId', 'credentialSubject.chainId'], + anyOf: [8996] + }) + expect(sort).to.deep.equal([ + { paths: ['indexedMetadata.stats.orders'], order: 'asc' } + ]) + expect(page).to.deep.equal({ offset: 5, limit: 5 }) + }) + + it('searches the orders of a consumer', async () => { + const { body } = await run({ type: 'orders', filter: { consumer: '0xC' }, size: 10 }) + expect(body).to.deep.equal({ results: [{ id: '0xorder' }], totalResults: 1 }) + expect(orderFind.firstCall.args).to.deep.equal([ + { consumer: '0xC' }, + { offset: 0, limit: 10 } + ]) + expect(ddoFind.called).to.equal(false) + }) + + it('gives the tags of the assets that match', async () => { + const { body } = await run({ type: 'tags', filter: { chainIds: [1] }, size: 50 }) + expect(body).to.deep.equal({ + results: [{ tag: 'weather', count: 2 }], + totalResults: 1 + }) + expect(ddoTags.firstCall.args[1]).to.equal(50) + }) + + it('answers 400 for an Elasticsearch body, an unknown filter and a deep page', async () => { + for (const query of [ + { query: { match_all: {} } }, + { filter: { color: 'red' } }, + { page: 101, size: 100 }, + 'not an object' + ]) { + const { status } = await run(query) + expect(status.httpStatus).to.equal(400) + } + expect(ddoFind.called).to.equal(false) + }) + + it('answers 500 for another error', async () => { + ddoFind.rejects(new Error('database is down')) + const { status } = await run({}) + expect(status.httpStatus).to.equal(500) + }) + + it('the DDO state query gives the key to the store', async () => { + const response = await new QueryDdoStateHandler(node).handle({ + command: PROTOCOL_COMMANDS.QUERY, + query: { txId: '0xtx' } + }) + expect(response.status.httpStatus).to.equal(200) + expect(ddoStateFind.firstCall.args[0]).to.deep.equal({ txId: '0xtx' }) + }) +}) + +describe('Elasticsearch DDO store: one search over all DDO indices', () => { + let search: sinon.SinonStub + let ddoDb: ElasticsearchDdoDatabase + + beforeEach(() => { + search = sinon.stub().resolves({ + hits: { hits: [{ _id: 'did:op:1', _source: {} }], total: { value: 42 } }, + aggregations: { + tags0: { buckets: [{ key: 'weather', doc_count: 2 }] }, + tags1: { + buckets: [ + { key: 'weather', doc_count: 1 }, + { key: 'ml', doc_count: 1 } + ] + } + } + }) + // the constructor only keeps the config; create() would connect to Elasticsearch + ddoDb = new (ElasticsearchDdoDatabase as any)({ dbType: 'elasticsearch' }) + sinon.stub(ddoDb as any, 'client').resolves({ search }) + }) + + afterEach(() => sinon.restore()) + + it('searches the DDO indices only, with the page as a document offset', async () => { + const found = await ddoDb.find([], [], { offset: 40, limit: 20 }) + expect(found).to.deep.equal({ results: [{ id: 'did:op:1' }], totalResults: 42 }) + const request = search.firstCall.args[0] + expect(request.index).to.deep.equal(ddoDb.getSchemas().map((s) => s.index)) + expect(request).to.include({ from: 40, size: 20, track_total_hits: true }) + expect(request.query).to.deep.equal({ match_all: {} }) + }) + + it('sorts by the key, then by id; an exact value of a dynamic field uses .keyword', async () => { + await ddoDb.find( + [{ paths: ['indexedMetadata.nft.owner'], anyOf: ['0xabc'] }], + [{ paths: ['indexedMetadata.stats.symbol'], order: 'asc' }], + { offset: 0, limit: 1 } + ) + const request = search.firstCall.args[0] + expect(request.sort).to.deep.equal([ + { + 'indexedMetadata.stats.symbol.keyword': { + order: 'asc', + missing: '_last', + unmapped_type: 'keyword' + } + }, + { id: { order: 'asc' } } + ]) + expect(request.query.bool.filter[0]).to.deep.equal({ + terms: { 'indexedMetadata.nft.owner.keyword': ['0xabc'] } + }) + }) + + it('adds the tags of the v4 and v5 fields together', async () => { + expect(await ddoDb.tags([], 10)).to.deep.equal([ + { tag: 'weather', count: 3 }, + { tag: 'ml', count: 1 } + ]) + }) +}) diff --git a/src/test/unit/safeUrl.test.ts b/src/test/unit/safeUrl.test.ts index 6088a1fc3..9dd37c5dd 100644 --- a/src/test/unit/safeUrl.test.ts +++ b/src/test/unit/safeUrl.test.ts @@ -153,7 +153,7 @@ describe('safeUrl: synchronous host checks', () => { expect(checkHostnameSync('')).to.equal('URL has no host') for (const host of [ 'example.com', - 'typesense', + 'elasticsearch', '8.8.8.8', '[2606:4700:4700::1111]' ]) { @@ -185,7 +185,7 @@ describe('safeUrl: synchronous host checks', () => { expect(checkUrlHost(url, configWith()), url).to.equal(PRIVATE_URL_ERROR) } expect(checkUrlHost('https://example.com/a', configWith())).to.equal(null) - expect(checkUrlHost('http://typesense:8108/', configWith())).to.equal(null) + expect(checkUrlHost('http://elasticsearch:9200/', configWith())).to.equal(null) expect(checkUrlHost('not a url', configWith())).to.equal('Invalid URL') expect(checkUrlHost('http://10.0.0.1/', configWith(['10.0.0.0/8']))).to.equal(null) }) @@ -240,7 +240,7 @@ describe('safeUrl: storage validation', () => { it('UrlStorage still accepts public URLs and DNS names (checked at connect time)', () => { for (const url of [ 'https://example.com/file.json', - 'http://typesense:8108/', + 'http://elasticsearch:9200/', 'http://172.15.0.7:80/' ]) { const storage = new UrlStorage({ type: 'url', url, method: 'get' }, config) diff --git a/src/test/unit/service/serviceDiskUsage.test.ts b/src/test/unit/service/serviceDiskUsage.test.ts index d8a9eac76..3608008f6 100644 --- a/src/test/unit/service/serviceDiskUsage.test.ts +++ b/src/test/unit/service/serviceDiskUsage.test.ts @@ -164,7 +164,9 @@ describe('service disk quota enforcement', () => { await clock.tickAsync(0) await clock.tickAsync(15_001) await Promise.all([...engine.serviceDiskChecksInFlight.values()]) - expect(timeout.calledOnce).to.equal(true) + // not calledOnce: in a full run, work that an earlier suite left can also ask for a + // timeout signal while the stub is in place + expect(timeout.called).to.equal(true) expect(engine.doStopService.called).to.equal(false) expect(engine.serviceOpsInFlight.size).to.equal(0) }) diff --git a/src/test/unit/service/serviceJobsDatabase.test.ts b/src/test/unit/service/serviceJobsDatabase.test.ts index 3ae97e754..7d35e2721 100644 --- a/src/test/unit/service/serviceJobsDatabase.test.ts +++ b/src/test/unit/service/serviceJobsDatabase.test.ts @@ -1,14 +1,18 @@ import { assert, expect } from 'chai' import { Readable } from 'stream' +import { Client } from '@elastic/elasticsearch' import { C2DDatabase } from '../../../components/database/C2DDatabase.js' -import { typesenseSchemas } from '../../../components/database/TypesenseSchemas.js' +import { Database } from '../../../components/database/index.js' +import { createElasticsearchClientWithRetry } from '../../../components/database/ElasticsearchConfigHelper.js' +import { stateSchemas } from '../../../components/database/ElasticSchemas.js' import { getConfiguration } from '../../../utils/config.js' import { buildEnvOverrideConfig, OverrideEnvConfig, setupEnvironment, tearDownEnvironment, - TEST_ENV_CONFIG_FILE + TEST_ENV_CONFIG_FILE, + describeOnElasticsearch } from '../../utils/utils.js' import { ENVIRONMENT_VARIABLES } from '../../../utils/constants.js' import { OceanNodeConfig } from '../../../@types/OceanNode.js' @@ -120,10 +124,11 @@ function makeServiceJob(overrides: Partial = {}): ServiceJob { } } -describe('Service Jobs Database', () => { +describeOnElasticsearch('Service Jobs Database', () => { let envOverrides: OverrideEnvConfig[] let config: OceanNodeConfig let db: C2DDatabase = null + let client: Client before(async () => { envOverrides = buildEnvOverrideConfig( @@ -134,7 +139,26 @@ describe('Service Jobs Database', () => { ) envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) config = await getConfiguration(true) - db = await new C2DDatabase(config.dbConfig, typesenseSchemas.c2dSchemas) + db = (await Database.init(config.dbConfig)).c2d + client = await createElasticsearchClientWithRetry(config.dbConfig) + }) + + // a lock release deletes without refresh, so refresh first: the wipe then sees no + // deleted locks + beforeEach(async () => { + const index = [ + stateSchemas.c2dJobs.index, + stateSchemas.serviceJobs.index, + stateSchemas.dockerImages.index, + stateSchemas.serviceLocks.index + ] + await client.indices.refresh({ index }) + await client.deleteByQuery({ + index, + query: { match_all: {} }, + refresh: true, + conflicts: 'proceed' + }) }) after(async () => { @@ -201,6 +225,63 @@ describe('Service Jobs Database', () => { ).to.be.lessThan(B_STALE_MS) await db.releaseServiceLock(serviceId, 'proc-A') }) + + it('B takes over after staleMs, A cannot release it, a refresh keeps it fresh', async () => { + const serviceId = `lock-svc-${Date.now()}-e` + // a lock write waits for the index refresh (up to 1 s), so the short window must + // be long enough that the refreshed stamp is still fresh when it is read back + const SHORT_STALE_MS = 1000 + const age = () => + new Promise((resolve) => setTimeout(resolve, SHORT_STALE_MS + 100)) + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(true) + expect(await db.acquireServiceLock(serviceId, 'proc-B', STALE_MS)).to.equal(false) + await age() + expect(await db.acquireServiceLock(serviceId, 'proc-B', SHORT_STALE_MS)).to.equal( + true + ) + await db.releaseServiceLock(serviceId, 'proc-A') + expect(await db.isServiceLocked(serviceId, STALE_MS)).to.equal(true) + await age() + expect(await db.isServiceLocked(serviceId, SHORT_STALE_MS)).to.equal(false) + await db.refreshServiceLocks('proc-B') + expect(await db.isServiceLocked(serviceId, SHORT_STALE_MS)).to.equal(true) + await db.releaseServiceLock(serviceId, 'proc-B') + expect(await db.isServiceLocked(serviceId, STALE_MS)).to.equal(false) + }) + + it('two holders racing for one stale lock: exactly one wins', async () => { + const serviceId = `lock-svc-${Date.now()}-f` + const SHORT_STALE_MS = 300 + expect(await db.acquireServiceLock(serviceId, 'proc-A', SHORT_STALE_MS)).to.equal( + true + ) + await new Promise((resolve) => setTimeout(resolve, SHORT_STALE_MS + 100)) + const results = await Promise.all([ + db.acquireServiceLock(serviceId, 'proc-B', SHORT_STALE_MS), + db.acquireServiceLock(serviceId, 'proc-C', SHORT_STALE_MS) + ]) + expect(results.filter(Boolean)).to.have.length(1) + }) + + it('a heartbeat between the read and the delete of a release does not keep the lock', async () => { + const serviceId = `lock-svc-${Date.now()}-g` + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(true) + // the first delete runs after a heartbeat changed the lock: it gets a 409 + const realDelete = client.delete + let deletes = 0 + ;(client as any).delete = async (params: any) => { + deletes++ + if (deletes === 1) await db.refreshServiceLocks('proc-A') + return realDelete.call(client, params) + } + try { + await db.releaseServiceLock(serviceId, 'proc-A') + } finally { + ;(client as any).delete = realDelete + } + expect(deletes).to.equal(2) + expect(await db.isServiceLocked(serviceId, STALE_MS)).to.equal(false) + }) }) it('inserts and reads back a service job by serviceId', async () => { @@ -224,6 +305,19 @@ describe('Service Jobs Database', () => { expect(aliceJobs.find((j) => j.serviceId === mine.serviceId)).to.not.equal(undefined) }) + it('finds a checksummed owner by its lower-cased address', async () => { + const owner = '0xe2DD09d719Da89e5a3D0F2549c7E24566e947260' + const job = makeServiceJob({ owner }) + await db.newServiceJob(job) + const found = await db.getServiceJob(undefined, owner.toLowerCase()) + expect(found.map((j) => j.serviceId)).to.deep.equal([job.serviceId]) + expect(found[0].owner).to.equal(owner) + const byId = await db.getServiceJob(job.serviceId, owner.toLowerCase()) + expect(byId.map((j) => j.serviceId)).to.deep.equal([job.serviceId]) + expect(await db.getServiceJob(job.serviceId, '0xsomeoneelse')).to.deep.equal([]) + expect(await db.getServiceJob('missing-service')).to.deep.equal([]) + }) + it('updates a service job (status + expiresAt + body)', async () => { const job = makeServiceJob() await db.newServiceJob(job) @@ -289,16 +383,21 @@ describe('Service Jobs Database', () => { status: ServiceStatusNumber.Running, clusterHash: 'other-cluster' }) - await db.newServiceJob(running) - await db.newServiceJob(starting) - await db.newServiceJob(locking) - await db.newServiceJob(claiming) - await db.newServiceJob(restarting) - await db.newServiceJob(error) - await db.newServiceJob(stopped) - await db.newServiceJob(unpaidError) - await db.newServiceJob(refundedStopped) - await db.newServiceJob(otherCluster) + // one refresh for all writes: each insert waits for it + await Promise.all( + [ + running, + starting, + locking, + claiming, + restarting, + error, + stopped, + unpaidError, + refundedStopped, + otherCluster + ].map((job) => db.newServiceJob(job)) + ) const active = await db.getRunningServiceJobs(CLUSTER_HASH) const ids = active.map((j) => j.serviceId) @@ -333,13 +432,12 @@ describe('Service Jobs Database', () => { status: ServiceStatusNumber.Starting, clusterHash: 'other-cluster' }) - await db.newServiceJob(starting) - await db.newServiceJob(locking) - await db.newServiceJob(claiming) - await db.newServiceJob(restarting) - await db.newServiceJob(running) - await db.newServiceJob(stopped) - await db.newServiceJob(otherCluster) + // one refresh for all writes: each insert waits for it + await Promise.all( + [starting, locking, claiming, restarting, running, stopped, otherCluster].map( + (job) => db.newServiceJob(job) + ) + ) const pending = await db.getPendingServiceStarts(CLUSTER_HASH) const ids = pending.map((j) => j.serviceId) @@ -374,11 +472,12 @@ describe('Service Jobs Database', () => { status: ServiceStatusNumber.Stopped, expiresAt: Date.now() + 3600_000 }) - await db.newServiceJob(expired) - await db.newServiceJob(expiredError) - await db.newServiceJob(future) - await db.newServiceJob(expiredStopped) - await db.newServiceJob(futureStopped) + // one refresh for all writes: each insert waits for it + await Promise.all( + [expired, expiredError, future, expiredStopped, futureStopped].map((job) => + db.newServiceJob(job) + ) + ) const expiredJobs = await db.getExpiredServiceJobs(CLUSTER_HASH) const ids = expiredJobs.map((j) => j.serviceId) @@ -424,9 +523,11 @@ describe('Service Jobs Database', () => { expiresAt: Date.now() - 10_000, outputArchives: [archive] }) - for (const job of [withArchives, cleaned, neverArchived, tooRecent, notExpired]) { - await db.newServiceJob(job) - } + await Promise.all( + [withArchives, cleaned, neverArchived, tooRecent, notExpired].map((job) => + db.newServiceJob(job) + ) + ) const jobs = await db.getExpiredServiceJobsBefore(Date.now() - 5_000, CLUSTER_HASH) const ids = jobs.map((j) => j.serviceId) @@ -488,7 +589,7 @@ describe('Service Jobs Database', () => { describe('shared resource accounting (compute + service)', () => { let engine: SharedAccountingEngine - before(async () => { + before(() => { engine = new SharedAccountingEngine(db) engine.setLimits( new Map([ @@ -496,13 +597,6 @@ describe('Service Jobs Database', () => { ['gpu0', 2] ]) ) - // Clean slate: retire any leftover jobs from earlier tests by marking them Expired — - // the only status that releases the reservation (Stopped still counts as active). - const leftovers = await db.getRunningServiceJobs(CLUSTER_HASH) - for (const j of leftovers) { - j.status = ServiceStatusNumber.Expired - await db.updateServiceJob(j) - } }) it('a running service occupies fungible resources in its own env', async () => { @@ -521,7 +615,14 @@ describe('Service Jobs Database', () => { }) it('fungible usage is NOT counted against a different env', async () => { - // The cpu service above is bound to 'env-shared'; a different env sees 0 cpu used. + await db.newServiceJob( + makeServiceJob({ + environment: 'env-shared', + status: ServiceStatusNumber.Running, + resources: [{ id: 'cpu', amount: 3 }] + }) + ) + // The cpu service is bound to 'env-shared'; a different env sees 0 cpu used. const otherEnv = makeEnv('env-other', [ { id: 'cpu', kind: 'fungible', total: 10, min: 1, max: 10 } as ComputeResource ]) @@ -553,7 +654,14 @@ describe('Service Jobs Database', () => { }) it('checkIfResourcesAreAvailable blocks a compute request when a service holds the GPU', async () => { - // gpu0 total:1 in the env, already 1 in use by the service above (global discrete). + await db.newServiceJob( + makeServiceJob({ + environment: 'env-gpu', + status: ServiceStatusNumber.Running, + resources: [{ id: 'gpu0', amount: 1 }] + }) + ) + // gpu0 total:1 in the env, already 1 in use by the service (global discrete). const env = makeEnv('env-elsewhere', [ { id: 'gpu0', @@ -622,6 +730,7 @@ describe('Service Jobs Database', () => { it('is a no-op when status changed (guards against clobbering a lifecycle transition)', async () => { const job = makeServiceJob({ status: ServiceStatusNumber.Running }) await db.newServiceJob(job) + const [inserted] = await db.getServiceJob(job.serviceId) const wrote = await db.updateServiceJobMetrics( job.serviceId, @@ -637,6 +746,8 @@ describe('Service Jobs Database', () => { const [fresh] = await db.getServiceJob(job.serviceId, job.owner) expect(fresh.runtimeMetrics).to.equal(undefined) + expect(fresh.status).to.equal(ServiceStatusNumber.Running) + expect(fresh.updatedAt).to.equal(inserted.updatedAt) }) it('is a no-op when the container was replaced (restart)', async () => { diff --git a/src/test/unit/service/serviceRestartRace.test.ts b/src/test/unit/service/serviceRestartRace.test.ts index c029b38fe..2f80395f8 100644 --- a/src/test/unit/service/serviceRestartRace.test.ts +++ b/src/test/unit/service/serviceRestartRace.test.ts @@ -556,7 +556,9 @@ describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { it('lease acquisition fails CLOSED when the lock DB errors (no docker/DB mutations)', async () => { const engine = makeEngine() - engine.db.acquireServiceLock = sinon.stub().rejects(new Error('SQLITE_BUSY')) + engine.db.acquireServiceLock = sinon + .stub() + .rejects(new Error('Elasticsearch unavailable')) try { await engine.restartService(SERVICE_ID, OWNER) diff --git a/src/test/unit/sqliteClient.test.ts b/src/test/unit/sqliteClient.test.ts deleted file mode 100644 index 6425b3a89..000000000 --- a/src/test/unit/sqliteClient.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { expect } from 'chai' -import fs from 'fs' -import os from 'os' -import path from 'path' -import { SqliteClient } from '../../components/database/sqliteClient.js' - -describe('SqliteClient', () => { - let tmpDir: string - let client: SqliteClient - - before(() => { - // Nested subdir on purpose: exercises the client's eager mkdir of the parent dir. - tmpDir = path.join( - os.tmpdir(), - `ocean-node-sqliteclient-${process.pid}-${Date.now()}`, - 'nested' - ) - client = new SqliteClient(path.join(tmpDir, 'test.sqlite')) - client.exec(` - CREATE TABLE IF NOT EXISTS kv ( - id TEXT PRIMARY KEY, - n INTEGER, - flag INTEGER, - blob BLOB - ); - `) - }) - - after(() => { - // best-effort cleanup of the temp DB dir (incl. any -wal/-shm siblings) - fs.rmSync(path.dirname(tmpDir), { recursive: true, force: true }) - }) - - it('creates the parent directory eagerly', () => { - expect(fs.existsSync(tmpDir)).to.equal(true) - }) - - it('run() reports changes: 1 on a hit, 0 on a miss', () => { - const inserted = client.run('INSERT INTO kv (id, n) VALUES (?, ?)', ['a', 1]) - expect(inserted.changes).to.equal(1) - expect(inserted.changes).to.be.a('number') - - const hit = client.run('UPDATE kv SET n = ? WHERE id = ?', [2, 'a']) - expect(hit.changes).to.equal(1) - - const miss = client.run('UPDATE kv SET n = ? WHERE id = ?', [9, 'does-not-exist']) - expect(miss.changes).to.equal(0) - }) - - it('get() returns undefined when no row matches', () => { - const row = client.get('SELECT * FROM kv WHERE id = ?', ['missing']) - expect(row).to.equal(undefined) - }) - - it('all() always returns an array', () => { - const rows = client.all('SELECT * FROM kv WHERE id = ?', ['still-missing']) - expect(rows).to.be.an('array').with.lengthOf(0) - }) - - it('sanitizes undefined -> NULL and boolean -> 1/0 bindings', () => { - // undefined would otherwise throw ERR_INVALID_ARG_TYPE in node:sqlite - client.run('INSERT INTO kv (id, n, flag) VALUES (?, ?, ?)', [ - 'sanitize', - undefined, - true - ]) - const row = client.get<{ n: number | null; flag: number }>( - 'SELECT n, flag FROM kv WHERE id = ?', - ['sanitize'] - ) - expect(row?.n).to.equal(null) - expect(row?.flag).to.equal(1) - - client.run('INSERT INTO kv (id, flag) VALUES (?, ?)', ['sanitize-false', false]) - const row2 = client.get<{ flag: number }>('SELECT flag FROM kv WHERE id = ?', [ - 'sanitize-false' - ]) - expect(row2?.flag).to.equal(0) - }) - - it('round-trips a BLOB written as Buffer (comes back as Uint8Array)', () => { - const original = JSON.stringify({ hello: 'world', n: 123 }) - client.run('INSERT INTO kv (id, blob) VALUES (?, ?)', ['blob', Buffer.from(original)]) - const row = client.get<{ blob: Uint8Array }>('SELECT blob FROM kv WHERE id = ?', [ - 'blob' - ]) - expect(row?.blob).to.be.an.instanceOf(Uint8Array) - // This is exactly the decode the compute provider must do: Buffer.from(blob).toString() - expect(Buffer.from(row!.blob).toString()).to.equal(original) - expect(JSON.parse(Buffer.from(row!.blob).toString())).to.deep.equal({ - hello: 'world', - n: 123 - }) - }) - - it('supports INSERT ... ON CONFLICT upsert (nonce/config pattern)', () => { - const sql = ` - INSERT INTO kv (id, n) VALUES (?, ?) - ON CONFLICT(id) DO UPDATE SET n = excluded.n; - ` - client.run(sql, ['upsert', 10]) - expect( - client.get<{ n: number }>('SELECT n FROM kv WHERE id = ?', ['upsert'])?.n - ).to.equal(10) - client.run(sql, ['upsert', 20]) - expect( - client.get<{ n: number }>('SELECT n FROM kv WHERE id = ?', ['upsert'])?.n - ).to.equal(20) - }) -}) diff --git a/src/test/unit/sqliteNodeMetrics.test.ts b/src/test/unit/sqliteNodeMetrics.test.ts deleted file mode 100644 index 0306cc222..000000000 --- a/src/test/unit/sqliteNodeMetrics.test.ts +++ /dev/null @@ -1,292 +0,0 @@ -import { expect } from 'chai' -import fs from 'fs' -import path from 'path' -import os from 'os' -import { - NodeMetricsDatabase, - floorToHour -} from '../../components/database/sqliteNodeMetrics.js' -import type { NodeMetricsSnapshot } from '../../@types/nodeMetrics.js' - -function baseSnapshot(overrides: Partial = {}): NodeMetricsSnapshot { - return { - collectedAt: Date.now(), - hasAggregate: true, - cpu: { - usagePercent: 10, - coresAllocated: 2, - hostCores: 8, - throttledCount: 0, - loadAverage: [0.1, 0.2, 0.3] - }, - memory: { - usedBytes: 100, - limitBytes: 1000, - hostFreeBytes: 500, - hostTotalBytes: 2000 - }, - disk: { usedBytes: 300 }, - network: { rxBytes: 10, txBytes: 20 }, - jobs: { running: 1, runningFree: 0, queued: 0, queuedFree: 0 }, - gpu: [], - env: [], - meta: { sampledContainers: 1, oldestSampleAgeSeconds: 5 }, - ...overrides - } -} - -describe('sqliteNodeMetrics.floorToHour', () => { - it('floors to the UTC hour boundary', () => { - const t = Date.UTC(2026, 6, 28, 10, 37, 42, 123) // 2026-07-28T10:37:42.123Z - const floored = floorToHour(t) - expect(floored).to.equal(Date.UTC(2026, 6, 28, 10, 0, 0, 0)) - }) - - it('is idempotent on an already-floored hour', () => { - const t = Date.UTC(2026, 6, 28, 10, 0, 0, 0) - expect(floorToHour(t)).to.equal(t) - }) -}) - -describe('NodeMetricsDatabase (temp SQLite file)', () => { - let dbPath: string - let db: NodeMetricsDatabase - - beforeEach(() => { - dbPath = path.join( - fs.mkdtempSync(path.join(os.tmpdir(), 'node-metrics-test-')), - 'nodeMetrics.sqlite' - ) - db = new NodeMetricsDatabase(dbPath) - }) - - afterEach(() => { - fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }) - }) - - it('rollupHour averages scalars and records sampleCount', () => { - const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) - db.insertSample( - baseSnapshot({ - collectedAt: hour + 60_000, - cpu: { ...baseSnapshot().cpu, usagePercent: 10 } - }) - ) - db.insertSample( - baseSnapshot({ - collectedAt: hour + 120_000, - cpu: { ...baseSnapshot().cpu, usagePercent: 30 } - }) - ) - const rolled = db.rollupHour(hour) - expect(rolled).to.equal(true) - - const rows = db.getHourly(hour, hour, 10) - expect(rows).to.have.length(1) - expect(rows[0].hourStart).to.equal(hour) - expect(rows[0].sampleCount).to.equal(2) - expect(rows[0].cpu.usagePercent).to.equal(20) // mean(10, 30) - }) - - it('rollupHour on an empty hour returns false and writes no row', () => { - const hour = Date.UTC(2026, 6, 28, 11, 0, 0, 0) - const rolled = db.rollupHour(hour) - expect(rolled).to.equal(false) - expect(db.getHourly(hour, hour, 10)).to.have.length(0) - }) - - it('rollupHour only consumes samples within [hourStart, hourStart+1h) — partial-hour boundary', () => { - const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) - const nextHour = hour + 60 * 60 * 1000 - db.insertSample(baseSnapshot({ collectedAt: hour + 1000 })) // inside hour - db.insertSample(baseSnapshot({ collectedAt: nextHour + 1000 })) // next hour, must not roll in - db.rollupHour(hour) - // the sample belonging to the next hour must still be present (not consumed) - const pending = db.getPendingSampleHours(nextHour + 60 * 60 * 1000) - expect(pending).to.include(nextHour) - expect(pending).to.not.include(hour) - }) - - it('rollupHour deletes the consumed raw samples after averaging', () => { - const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) - db.insertSample(baseSnapshot({ collectedAt: hour + 1000 })) - db.rollupHour(hour) - // nothing left in that hour to roll up again (idempotent-ish: no samples, so returns false) - expect(db.rollupHour(hour)).to.equal(false) - }) - - it('rollupHour is upsert: re-rolling an hourStart overwrites rather than duplicates', () => { - const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) - db.insertSample( - baseSnapshot({ - collectedAt: hour + 1000, - cpu: { ...baseSnapshot().cpu, usagePercent: 10 } - }) - ) - db.rollupHour(hour) - db.insertSample( - baseSnapshot({ - collectedAt: hour + 2000, - cpu: { ...baseSnapshot().cpu, usagePercent: 90 } - }) - ) - db.rollupHour(hour) - const rows = db.getHourly(hour, hour, 10) - expect(rows).to.have.length(1) - expect(rows[0].cpu.usagePercent).to.equal(90) - expect(rows[0].sampleCount).to.equal(1) - }) - - it('averages GPU metrics per resourceId and env resources per env+resource', () => { - const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) - db.insertSample( - baseSnapshot({ - collectedAt: hour + 1000, - gpu: [ - { - resourceId: 'gpu0', - vendor: 'nvidia', - utilizationPercent: 20, - memoryUsedBytes: 100 - } - ], - env: [{ env: 'envA', resource: 'cpu', total: 4, inUse: 2 }] - }) - ) - db.insertSample( - baseSnapshot({ - collectedAt: hour + 2000, - gpu: [ - { - resourceId: 'gpu0', - vendor: 'nvidia', - utilizationPercent: 80, - memoryUsedBytes: 300 - } - ], - env: [{ env: 'envA', resource: 'cpu', total: 4, inUse: 4 }] - }) - ) - db.rollupHour(hour) - const [row] = db.getHourly(hour, hour, 10) - expect(row.gpu).to.have.length(1) - expect(row.gpu[0].resourceId).to.equal('gpu0') - expect(row.gpu[0].utilizationPercent).to.equal(50) // mean(20, 80) - expect(row.gpu[0].memoryUsedBytes).to.equal(200) // mean(100, 300) - expect(row.env).to.have.length(1) - expect(row.env[0].inUse).to.equal(3) // mean(2, 4) - }) - - it('GPUs present in only some samples do not have their mean dragged down by absent samples', () => { - const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) - // gpu0 reports temperatureC only on the second sample - db.insertSample( - baseSnapshot({ - collectedAt: hour + 1000, - gpu: [{ resourceId: 'gpu0', utilizationPercent: 10 }] - }) - ) - db.insertSample( - baseSnapshot({ - collectedAt: hour + 2000, - gpu: [{ resourceId: 'gpu0', utilizationPercent: 30, temperatureC: 60 }] - }) - ) - db.rollupHour(hour) - const [row] = db.getHourly(hour, hour, 10) - expect(row.gpu[0].utilizationPercent).to.equal(20) // mean over 2 samples - expect(row.gpu[0].temperatureC).to.equal(60) // mean over 1 sample that reported it, not 30 - }) - - it('purgeOldSamples deletes samples older than the cutoff and returns the count', () => { - const now = Date.now() - db.insertSample(baseSnapshot({ collectedAt: now - 4 * 60 * 60 * 1000 })) // old - db.insertSample(baseSnapshot({ collectedAt: now })) // recent - const purged = db.purgeOldSamples(now - 3 * 60 * 60 * 1000) - expect(purged).to.equal(1) - }) - - it('deleteHourlyOlderThan enforces retention and returns the count deleted', () => { - const oldHour = Date.UTC(2020, 0, 1) - const recentHour = floorToHour(Date.now()) - db.insertSample(baseSnapshot({ collectedAt: oldHour + 1000 })) - db.rollupHour(oldHour) - db.insertSample(baseSnapshot({ collectedAt: recentHour + 1000 })) - db.rollupHour(recentHour) - - const deleted = db.deleteHourlyOlderThan(Date.now() - 180 * 24 * 60 * 60 * 1000) - expect(deleted).to.equal(1) - const remaining = db.getHourly(0, Date.now(), 1000) - expect(remaining).to.have.length(1) - expect(remaining[0].hourStart).to.equal(recentHour) - }) - - it('getHourly returns ordered buckets within range, capped at limit', () => { - const h0 = Date.UTC(2026, 6, 28, 8, 0, 0, 0) - const h1 = h0 + 60 * 60 * 1000 - const h2 = h1 + 60 * 60 * 1000 - for (const h of [h0, h1, h2]) { - db.insertSample(baseSnapshot({ collectedAt: h + 1000 })) - db.rollupHour(h) - } - const rows = db.getHourly(h0, h2, 2) - expect(rows).to.have.length(2) - expect(rows[0].hourStart).to.equal(h0) - expect(rows[1].hourStart).to.equal(h1) - }) - - it('JSON round-trips gpu/env through storage (safeParse recovers structured fields)', () => { - const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) - db.insertSample( - baseSnapshot({ - collectedAt: hour + 1000, - gpu: [{ resourceId: 'gpu0', vendor: 'nvidia', powerWatts: 88 }], - env: [{ env: 'envA', resource: 'gpu0', total: 1, inUse: 1 }] - }) - ) - db.rollupHour(hour) - const [row] = db.getHourly(hour, hour, 10) - expect(row.gpu[0]).to.include({ - resourceId: 'gpu0', - vendor: 'nvidia', - powerWatts: 88 - }) - expect(row.env[0]).to.deep.equal({ - env: 'envA', - resource: 'gpu0', - total: 1, - inUse: 1 - }) - }) - - it('getPartialHour averages the raw (un-rolled) samples and flags partial:true', () => { - const hour = Date.UTC(2026, 6, 28, 10, 0, 0, 0) - db.insertSample( - baseSnapshot({ - collectedAt: hour + 60_000, - cpu: { ...baseSnapshot().cpu, usagePercent: 10 } - }) - ) - db.insertSample( - baseSnapshot({ - collectedAt: hour + 120_000, - cpu: { ...baseSnapshot().cpu, usagePercent: 30 } - }) - ) - const partial = db.getPartialHour(hour) - expect(partial).to.not.equal(null) - expect(partial!.partial).to.equal(true) - expect(partial!.hourStart).to.equal(hour) - expect(partial!.sampleCount).to.equal(2) - expect(partial!.cpu.usagePercent).to.equal(20) // mean(10, 30) — same averaging as rollupHour - }) - - it('getPartialHour returns null when the hour has no samples, and does not consume samples', () => { - const hour = Date.UTC(2026, 6, 28, 12, 0, 0, 0) - expect(db.getPartialHour(hour)).to.equal(null) - // it is read-only: a sample in a populated hour survives a getPartialHour call - db.insertSample(baseSnapshot({ collectedAt: hour + 1000 })) - expect(db.getPartialHour(hour)!.sampleCount).to.equal(1) - expect(db.getPartialHour(hour)!.sampleCount).to.equal(1) - }) -}) diff --git a/src/test/utils/hooks.ts b/src/test/utils/hooks.ts index e676fa2f0..ed4f0eb14 100644 --- a/src/test/utils/hooks.ts +++ b/src/test/utils/hooks.ts @@ -3,7 +3,7 @@ // and after finishing them respectively. import type { Context as MochaContext } from 'mocha' -import { DB_TYPES, ENVIRONMENT_VARIABLES } from '../../utils/constants.js' +import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js' import { CONFIG_LOGGER } from '../../utils/logging/common.js' import { setupEnvironment, @@ -12,8 +12,7 @@ import { getExistingEnvironment, TEST_ENV_CONFIG_FILE, buildEnvOverrideConfig, - DEFAULT_TEST_TIMEOUT, - SELECTED_RUN_DATABASE + DEFAULT_TEST_TIMEOUT } from './utils.js' // current process.env environment @@ -34,22 +33,19 @@ function getEnvOverrides(): OverrideEnvConfig[] { ENVIRONMENT_VARIABLES.ARWEAVE_GATEWAY, ENVIRONMENT_VARIABLES.RPCS, ENVIRONMENT_VARIABLES.PRIVATE_KEY, - ENVIRONMENT_VARIABLES.DB_TYPE, - ENVIRONMENT_VARIABLES.DB_URL, // the test node's indexer calls its own HTTP API (decryptor) on loopback, // same as Barge's node.yml - ENVIRONMENT_VARIABLES.ALLOWED_PRIVATE_URL_RANGES + ENVIRONMENT_VARIABLES.ALLOWED_PRIVATE_URL_RANGES, + // the node needs a DB_TYPE; the CI legs set it, a local run gets SQLite + ENVIRONMENT_VARIABLES.DB_TYPE ], [ 'http://172.15.0.16:8080/', 'https://arweave.net/', '{ "8996": {"rpc": "http://127.0.0.1:8545", "chainId": 8996, "network": "development", "chunkSize": 100}, "137": {"rpc": "https://polygon.meowrpc.com", "chainId": 137, "network": "polygon", "chunkSize": 100 }}', '0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58', - SELECTED_RUN_DATABASE, - SELECTED_RUN_DATABASE === DB_TYPES.ELASTIC_SEARCH - ? 'http://localhost:9200' - : 'http://localhost:8108/?apiKey=xyz', - '["127.0.0.1/32"]' + '["127.0.0.1/32"]', + process.env.DB_TYPE || 'sqlite' ] ) } diff --git a/src/test/utils/sqlClient.ts b/src/test/utils/sqlClient.ts new file mode 100644 index 000000000..f036125ab --- /dev/null +++ b/src/test/utils/sqlClient.ts @@ -0,0 +1,45 @@ +import type { OceanNodeDBConfig } from '../../@types/OceanNode.js' +import { PostgresClient } from '../../components/database/sql/PostgresClient.js' +import type { SqlClient } from '../../components/database/sql/SqlClient.js' +import { SqliteClient } from '../../components/database/sql/SqliteClient.js' + +// The suites of the SQL stores run on SQLite in memory. With DB_TYPE=postgresql they run +// on the PostgreSQL server of DB_URL, DB_USERNAME and DB_PASSWORD. +export const SQL_DB_TYPE = process.env.DB_TYPE === 'postgresql' ? 'postgresql' : 'sqlite' +export const SQL_DB_CONFIG: OceanNodeDBConfig = { dbType: SQL_DB_TYPE } + +// read when the file loads: a test can change the DB_* variables for its own purpose +const POSTGRES_SERVER: OceanNodeDBConfig = { + dbType: 'postgresql', + url: process.env.DB_URL, + username: process.env.DB_USERNAME, + password: process.env.DB_PASSWORD +} + +let schemaCount = 0 + +// a client on a new, empty database; close() removes the database +export function makeSqlClient(): Promise { + return SQL_DB_TYPE === 'postgresql' + ? makePostgresClient() + : Promise.resolve(new SqliteClient(':memory:')) +} + +// Each client gets a schema of its own on the server, so a test never sees the tables of +// another test or of a node. +async function makePostgresClient(): Promise { + const config = POSTGRES_SERVER + const schema = `ocean_test_${process.pid}_${++schemaCount}` + const admin = new PostgresClient(config) + await admin.exec(`CREATE SCHEMA ${schema}`) + const url = new URL(config.url) + url.searchParams.set('options', `-c search_path=${schema}`) + const client = new PostgresClient({ ...config, url: url.toString() }) + const closeClient = client.close.bind(client) + client.close = async () => { + await closeClient() + await admin.exec(`DROP SCHEMA ${schema} CASCADE`) + await admin.close() + } + return client +} diff --git a/src/test/utils/utils.ts b/src/test/utils/utils.ts index d3153f40b..a9fcd97d5 100644 --- a/src/test/utils/utils.ts +++ b/src/test/utils/utils.ts @@ -1,12 +1,25 @@ +import type { Suite } from 'mocha' import path from 'path' import dotenv from 'dotenv' import { fileURLToPath } from 'url' -import { DB_TYPES, ENVIRONMENT_VARIABLES, EnvVariable } from '../../utils/constants.js' +import { ENVIRONMENT_VARIABLES, EnvVariable } from '../../utils/constants.js' import { CONFIG_LOGGER } from '../../utils/logging/common.js' import { RPCS } from '../../@types/blockchain.js' +import type { NodeMetricsSnapshot } from '../../@types/nodeMetrics.js' import { getConfiguration } from '../../utils/config/builder.js' export const DEFAULT_TEST_TIMEOUT = 20000 // 20 secs MAX + +// A suite that uses the Elasticsearch client directly (it reads or cleans an index) runs +// only when DB_TYPE selects Elasticsearch. The stores of the other databases have their own +// suites. +export function describeOnElasticsearch( + title: string, + suite: (this: Suite) => void +): void { + const run = process.env.DB_TYPE === 'elasticsearch' ? describe : describe.skip + run(title, suite) +} // __dirname and __filename are not defined in ES module scope const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) @@ -163,9 +176,34 @@ export function isRunningContinousIntegrationEnv(): boolean { ) } -// does a random run; sometimes elastic, others typesense -export const SELECTED_RUN_DATABASE = - new Date().getTime() % 2 === 0 ? DB_TYPES.ELASTIC_SEARCH : DB_TYPES.TYPESENSE -CONFIG_LOGGER.debug(`SELECTED_RUN_DATABASE: ${SELECTED_RUN_DATABASE}`) - export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +// a node metrics snapshot for the tests +export function baseSnapshot( + overrides: Partial = {} +): NodeMetricsSnapshot { + return { + collectedAt: Date.now(), + hasAggregate: true, + cpu: { + usagePercent: 10, + coresAllocated: 2, + hostCores: 8, + throttledCount: 0, + loadAverage: [0.1, 0.2, 0.3] + }, + memory: { + usedBytes: 100, + limitBytes: 1000, + hostFreeBytes: 500, + hostTotalBytes: 2000 + }, + disk: { usedBytes: 300 }, + network: { rxBytes: 10, txBytes: 20 }, + jobs: { running: 1, runningFree: 0, queued: 0, queuedFree: 0 }, + gpu: [], + env: [], + meta: { sampledContainers: 1, oldestSampleAgeSeconds: 5 }, + ...overrides + } +} diff --git a/src/utils/config/builder.ts b/src/utils/config/builder.ts index e475ceaa8..0b2801d54 100644 --- a/src/utils/config/builder.ts +++ b/src/utils/config/builder.ts @@ -1,4 +1,4 @@ -import type { OceanNodeConfig } from '../../@types/OceanNode.js' +import type { OceanNodeConfig, OceanNodeDBConfig } from '../../@types/OceanNode.js' import type { C2DClusterInfo, C2DDockerConfig } from '../../@types/C2D/C2D.js' import type { RPCS } from '../../@types/blockchain.js' import type { FeeTokens } from '../../@types/Fees.js' @@ -16,6 +16,7 @@ import { OCEAN_ARTIFACTS_ADDRESSES_PER_CHAIN } from '../address.js' import { create256Hash } from '../crypt.js' +import { dbConfigurationProblem } from '../database.js' import { CONFIG_LOGGER } from '../logging/common.js' import { redactUrls, safeStringify } from '../logging/redact.js' import { OceanNodeConfigSchema } from './schemas.js' @@ -27,6 +28,19 @@ import lodash from 'lodash' let previousConfiguration: OceanNodeConfig = null +// SQLite keeps its files in the node folder, so connection values have no effect on it +function withoutUnusedConnection(dbConfig: OceanNodeDBConfig): OceanNodeDBConfig { + if (dbConfig.dbType !== 'sqlite') { + return dbConfig + } + if (dbConfig.url) { + CONFIG_LOGGER.warn( + 'DB_URL is ignored: DB_TYPE is sqlite. Set DB_TYPE to postgresql or elasticsearch to use it' + ) + } + return { dbType: 'sqlite' } +} + function mapEnvToConfig( env: NodeJS.ProcessEnv, mapping: Record @@ -55,19 +69,27 @@ function preprocessConfigData(data: any): void { delete data.INTERFACES } - // Transform DB_* env vars to dbConfig - if (data.DB_URL) { - data.dbConfig = { + // Transform DB_* env vars to dbConfig. Only the variables that are set override the + // config file, so a file can hold the URL and the environment only the password. + const dbOverrides = lodash.omitBy( + { + dbType: data.DB_TYPE, url: data.DB_URL, username: data.DB_USERNAME, - password: data.DB_PASSWORD, - dbType: data.DB_TYPE || 'elasticsearch' - } - delete data.DB_URL - delete data.DB_USERNAME - delete data.DB_PASSWORD - delete data.DB_TYPE + password: data.DB_PASSWORD + }, + // an empty value (`DB_TYPE=${DB_TYPE}` in a compose file) is not a setting + (value) => value === undefined || value === '' + ) + // a `dbConfig` in the file that is not an object fails the validation + if (data.dbConfig === undefined || lodash.isPlainObject(data.dbConfig)) { + // no default and no guess from DB_URL: with no DB_TYPE the validation fails + data.dbConfig = { ...data.dbConfig, ...dbOverrides } } + delete data.DB_TYPE + delete data.DB_URL + delete data.DB_USERNAME + delete data.DB_PASSWORD // Transform FEE_* env vars to feeStrategy if (data.FEE_AMOUNT && data.FEE_TOKENS) { @@ -239,6 +261,12 @@ export function buildMergedConfig(): OceanNodeConfig { } const config = parsed.data as any + const dbProblem = dbConfigurationProblem(config.dbConfig) + if (dbProblem) { + CONFIG_LOGGER.error(dbProblem) + throw new Error('Configuration validation failed') + } + config.dbConfig = withoutUnusedConnection(config.dbConfig) // PERSISTENT_STORAGE_ALLOW_BUCKET_SHARING overrides persistentStorage.allowBucketSharing. // Applied after parsing because PERSISTENT_STORAGE may still be a JSON string before it. diff --git a/src/utils/config/constants.ts b/src/utils/config/constants.ts index fff50cb93..ef0fcc47f 100644 --- a/src/utils/config/constants.ts +++ b/src/utils/config/constants.ts @@ -1,12 +1,12 @@ export const ENV_TO_CONFIG_MAPPING = { PRIVATE_KEY: 'keys.privateKey', INTERFACES: 'INTERFACES', + DB_TYPE: 'DB_TYPE', DB_URL: 'DB_URL', DB_USERNAME: 'DB_USERNAME', DB_PASSWORD: 'DB_PASSWORD', - DB_TYPE: 'DB_TYPE', // NOTE: deliberately flat (not under dbConfig.*) — preprocessConfigData() rebuilds - // data.dbConfig from scratch when DB_URL is set, which would drop anything nested there. + // data.dbConfig from scratch when any DB_* is set, which would drop anything nested there. DB_INIT_MAX_ATTEMPTS: 'dbInitMaxAttempts', DB_INIT_RETRY_DELAY: 'dbInitRetryDelay', DB_INIT_MAX_RETRY_DELAY: 'dbInitMaxRetryDelay', @@ -28,6 +28,7 @@ export const ENV_TO_CONFIG_MAPPING = { ESCROW_CLAIM_TIMEOUT: 'claimDurationTimeout', VALIDATE_UNSIGNED_DDO: 'validateUnsignedDDO', JWT_SECRET: 'jwtSecret', + INSTANCE_ID: 'instanceId', MAX_REQ_PER_MINUTE: 'rateLimit', MAX_CONNECTIONS_PER_MINUTE: 'maxConnections', TRUST_PROXY: 'trustProxy', diff --git a/src/utils/config/schemas.ts b/src/utils/config/schemas.ts index d759d29b1..8cec3f665 100644 --- a/src/utils/config/schemas.ts +++ b/src/utils/config/schemas.ts @@ -1,6 +1,6 @@ import { z } from 'zod' import { getAddress } from 'ethers' -import { dhtFilterMethod } from '../../@types/OceanNode.js' +import { DATABASE_TYPES, dhtFilterMethod } from '../../@types/OceanNode.js' import { C2DClusterType } from '../../@types/C2D/C2D.js' import { DEFAULT_SERVICE_MAX_DURATION_SECONDS, @@ -265,10 +265,12 @@ export const FeeStrategySchema = z.object({ }) export const OceanNodeDBConfigSchema = z.object({ - url: z.string().nullable(), + dbType: z.enum(DATABASE_TYPES, { + error: `DB_TYPE is required and must be one of: ${DATABASE_TYPES.join(', ')}` + }), + url: z.string().optional(), username: z.string().optional(), - password: z.string().optional(), - dbType: z.string().nullable() + password: z.string().optional() }) export const PersistentStorageConfigSchema = z @@ -1506,7 +1508,7 @@ export const OceanNodeConfigSchema = z DB_USERNAME: z.string().optional(), DB_PASSWORD: z.string().optional(), DB_TYPE: z.string().optional(), - dbConfig: OceanNodeDBConfigSchema.optional(), + dbConfig: OceanNodeDBConfigSchema, // Accept either an object (config file) or a JSON string (env var `PERSISTENT_STORAGE`), // and validate the parsed value against the PersistentStorage schema. persistentStorage: z @@ -1746,6 +1748,7 @@ export const OceanNodeConfigSchema = z isBootstrap: booleanFromString.optional().default(false), validateUnsignedDDO: booleanFromString.optional().default(true), jwtSecret: z.string(), + instanceId: z.string().optional(), httpCertPath: z.string().optional(), httpKeyPath: z.string().optional() }) diff --git a/src/utils/constants.ts b/src/utils/constants.ts index c8966d496..0d61d3003 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -137,7 +137,7 @@ export const SUPPORTED_PROTOCOL_COMMANDS: string[] = [ // Maintenance (drain) mode. When enabled, the node refuses NEW commitment (compute/service // starts, initialize, extend) but keeps serving already-committed work. The flag persists in -// the SQLite config DB under this key and is cached in memory on the OceanNode singleton. +// the config DB under this key and is cached in memory on the OceanNode singleton. export const MAINTENANCE_MODE_CONFIG_KEY = 'maintenanceMode' // Stable machine-readable code clients discriminate on. 503 is already reused in these handlers // for "subsystem unavailable" (e.g. "Compute engines not configured"), so the status alone is @@ -341,11 +341,6 @@ export interface EnvVariable { required: boolean } -export const DB_TYPES = { - ELASTIC_SEARCH: 'elasticsearch', - TYPESENSE: 'typesense' -} - // Every P2P_* env var is derived from ENV_TO_CONFIG_MAPPING (src/utils/config/constants.ts), // which is the single source of truth for what the config system actually reads. Generating // them instead of hand-listing means the two lists can no longer drift apart. @@ -390,7 +385,12 @@ export const ENVIRONMENT_VARIABLES: Record = { value: process.env.INDEXER_NETWORKS, required: false }, + // sqlite (default), postgresql or elasticsearch; URL, username and password are for the + // last two only + DB_TYPE: { name: 'DB_TYPE', value: process.env.DB_TYPE, required: false }, DB_URL: { name: 'DB_URL', value: process.env.DB_URL, required: false }, + DB_USERNAME: { name: 'DB_USERNAME', value: process.env.DB_USERNAME, required: false }, + DB_PASSWORD: { name: 'DB_PASSWORD', value: process.env.DB_PASSWORD, required: false }, // these 2 bellow will change in the future (not required, just remove functionality) IPFS_GATEWAY: { name: 'IPFS_GATEWAY', @@ -466,7 +466,7 @@ export const ENVIRONMENT_VARIABLES: Record = { }, MAINTENANCE_MODE: { // boot the node directly into maintenance (drain) mode. "true" enables it; the value is - // written through to the SQLite config DB and is STICKY across restarts — returning to + // written through to the config DB and is STICKY across restarts — returning to // normal service requires an explicit `setMaintenance { enabled: false }` admin command // AND unsetting this env before the next restart. name: 'MAINTENANCE_MODE', @@ -586,7 +586,7 @@ export const ENVIRONMENT_VARIABLES: Record = { required: false }, LOG_DB: { - // log to the DB (Typesense / Elasticsearch)? default false. falls back to files when the + // log to the DB? default false. falls back to files when the // log DB is not available name: 'LOG_DB', value: process.env.LOG_DB, @@ -634,11 +634,6 @@ export const ENVIRONMENT_VARIABLES: Record = { value: process.env.FILEINFO_MAX_REQ_PER_MINUTE, required: false }, - DB_TYPE: { - name: 'DB_TYPE', - value: process.env.DB_TYPE, - required: false - }, DB_INIT_MAX_ATTEMPTS: { name: 'DB_INIT_MAX_ATTEMPTS', value: process.env.DB_INIT_MAX_ATTEMPTS, @@ -838,13 +833,18 @@ export const ENVIRONMENT_VARIABLES: Record = { name: 'NODE_METRICS_RETENTION_DAYS', value: process.env.NODE_METRICS_RETENTION_DAYS, required: false + }, + INSTANCE_ID: { + name: 'INSTANCE_ID', + value: process.env.INSTANCE_ID, + required: false } } export const CONNECTION_HISTORY_DELETE_THRESHOLD = 300 // 1 minute export const CONNECTIONS_RATE_INTERVAL = 60 * 1000 -// Typesense's maximum limit to send 250 hits at a time -export const TYPESENSE_HITS_CAP = 250 +// page size for the client-side escrow range scan +export const ESCROW_SCAN_PAGE_SIZE = 250 // Upper bound on how many escrow rows the handler will scan from the store when a client-side // (BigInt) range filter is active. Keeps deep pagination bounded; beyond this the result is @@ -853,3 +853,7 @@ export const MAX_ESCROW_CLIENT_SCAN = 2000 export const DDO_IDENTIFIER_PREFIX = 'did:op:' // global ocean node API services path export const SERVICES_API_BASE_PATH = '/api/services' +// DDO routes: resolve, state, validate +export const DDO_API_BASE_PATH = '/api/ddo' +// the DDO, order and tag search: the path of the `query` command +export const QUERY_API_PATH = '/api/query' diff --git a/src/utils/cronjobs/nodeMetricsJobs.ts b/src/utils/cronjobs/nodeMetricsJobs.ts index 07db951ec..86b1f5ff7 100644 --- a/src/utils/cronjobs/nodeMetricsJobs.ts +++ b/src/utils/cronjobs/nodeMetricsJobs.ts @@ -12,7 +12,7 @@ import { collectNodeMetricsSnapshot, hasFreshAggregate } from '../../components/core/utils/nodeMetricsHandler.js' -import { floorToHour } from '../../components/database/sqliteNodeMetrics.js' +import { HOUR_MS, floorToHour } from '../../components/database/nodeMetricsAggregate.js' import { getMetricsRetentionDays, isNodeMetricsHistoryEnabled @@ -22,7 +22,6 @@ import { // which the request handler imports directly to avoid depending on this scheduler module. export { getMetricsRetentionDays, isNodeMetricsHistoryEnabled } -const HOUR_MS = 60 * 60 * 1000 const DEFAULT_SAMPLE_CRON = '* * * * *' // Raw minute samples are consumed by the hourly roll-up; this is only the safety window after // which any leftover samples are purged unconditionally. @@ -51,9 +50,10 @@ export function scheduleNodeMetricsJobs(node: OceanNode, dbconn: Database | null return } const db = dbconn.nodeMetrics + const instanceId = node.getInstanceId() // 1. Minute sampler - scheduleCron(getSampleCronExpression(), () => { + scheduleCron(getSampleCronExpression(), async () => { try { // Skip when there is no fresh compute aggregate (metrics collection disabled or no // engine has sampled yet). Persisting all-zero rows would skew hourly averages. @@ -68,7 +68,7 @@ export function scheduleNodeMetricsJobs(node: OceanNode, dbconn: Database | null return } const snapshot = collectNodeMetricsSnapshot(node) - db.insertSample(snapshot) + await db.insertSample(instanceId, snapshot) OCEAN_NODE_LOGGER.resetThrottle(SAMPLER_ERROR_THROTTLE_KEY) } catch (err) { OCEAN_NODE_LOGGER.throttled( @@ -81,16 +81,16 @@ export function scheduleNodeMetricsJobs(node: OceanNode, dbconn: Database | null }) // 2. Hourly roll-up at :05 — average each complete hour's samples, then drop consumed rows. - scheduleCron('5 * * * *', () => { + scheduleCron('5 * * * *', async () => { try { const currentHour = floorToHour(Date.now()) - const pending = db.getPendingSampleHours(currentHour) + const pending = await db.getPendingSampleHours(instanceId, currentHour) let rolled = 0 for (const hour of pending) { - if (db.rollupHour(hour)) rolled++ + if (await db.rollupHour(instanceId, hour)) rolled++ } // Safety net for any orphaned samples the roll-up did not consume. - const purged = db.purgeOldSamples(Date.now() - SAMPLE_BUFFER_MS) + const purged = await db.purgeOldSamples(instanceId, Date.now() - SAMPLE_BUFFER_MS) if (rolled > 0 || purged > 0) { OCEAN_NODE_LOGGER.debug( `Node metrics roll-up: ${rolled} hour(s) aggregated, ${purged} stale sample(s) purged.` @@ -104,10 +104,10 @@ export function scheduleNodeMetricsJobs(node: OceanNode, dbconn: Database | null }) // 3. Daily retention sweep at 03:07. - scheduleCron('7 3 * * *', () => { + scheduleCron('7 3 * * *', async () => { try { const cutoff = Date.now() - getMetricsRetentionDays() * 24 * HOUR_MS - const deleted = db.deleteHourlyOlderThan(cutoff) + const deleted = await db.deleteHourlyOlderThan(instanceId, cutoff) if (deleted > 0) { OCEAN_NODE_LOGGER.debug( `Node metrics retention: deleted ${deleted} hourly row(s) older than ${getMetricsRetentionDays()} days.` diff --git a/src/utils/cronjobs/p2pAnnounceDDOS.ts b/src/utils/cronjobs/p2pAnnounceDDOS.ts index 119558a39..ff8a98615 100644 --- a/src/utils/cronjobs/p2pAnnounceDDOS.ts +++ b/src/utils/cronjobs/p2pAnnounceDDOS.ts @@ -4,6 +4,8 @@ import { OceanNode } from '../../OceanNode.js' import { P2P_LOGGER } from '../logging/common.js' import { provideLimit } from '../../components/P2P/provideLimiter.js' +import type { Condition, SortSpec } from '../../components/database/BaseDatabase.js' +import { DDO_FIELDS } from '../../components/database/assetQuery.js' import { classifyP2PError, logProvideFailure, @@ -11,116 +13,11 @@ import { } from '../../components/P2P/errors.js' /** - * How many documents to ask each schema for per round trip. Kept well under the Typesense - * per-page cap (250) so the same number is legal on both backends, and small enough that a - * page's worth of provides drains through the shared limiter before the next page is fetched. + * How many documents to ask for per round trip. Small enough that a page's worth of provides + * drains through the shared limiter before the next page is fetched. */ const REPUBLISH_PAGE_SIZE = 100 - -/** - * A stop so a backend that keeps answering with full pages - a paging bug, a store that grows - * while we walk it - cannot turn startup into an unbounded loop. At the page size above this - * is a million documents, far past any real node. - */ -const MAX_REPUBLISH_PAGES = 10_000 - -interface DdoPage { - documents: any[] - /** - * True when at least one schema filled its page, which is the only signal either backend - * gives that there may be more. Neither returns a reliable total across all schemas: the - * Elasticsearch path returns bare document arrays with no count at all, and the Typesense - * path returns a per-collection `found` that says nothing about the other collections. - */ - hasFullBatch: boolean -} - -/** - * The two DDO backends answer a search with different shapes, and the republish has to read - * both: - * - * - Elasticsearch returns one plain array of documents per schema that had hits (schemas with - * no hits are simply absent), each document already flattened out of `hits.hits[]._source`. - * - Typesense returns one response object per collection, always, each with `found` (that - * collection's total) and `hits[]`, where the document sits under `hit.document`. - * - * Reading only `result[0].found` / `result[0].hits` - a Typesense-only shape - meant the - * Elasticsearch path never iterated at all, and the Typesense path only ever looked at the - * first collection, which for a node holding current DDOs is the empty deprecated-DDO one. - */ -function extractPage(result: any, pageSize: number): DdoPage { - const documents: any[] = [] - let hasFullBatch = false - if (!Array.isArray(result)) { - return { documents, hasFullBatch } - } - for (const perSchema of result) { - // The raw page size decides whether the page was full - measured before filtering, because a - // page that came back full but held documents without an id still means the store may have - // more pages. Testing the filtered count instead would drop `hasFullBatch` to false on such - // a page and stop the walk early, leaving later pages never re-provided. - let rawCount = 0 - let batch: any[] = [] - if (Array.isArray(perSchema)) { - rawCount = perSchema.length - batch = perSchema.filter((document: any) => document && document.id) - } else if (perSchema && Array.isArray(perSchema.hits)) { - rawCount = perSchema.hits.length - batch = perSchema.hits - .map((hit: any) => hit?.document) - .filter((document: any) => document && document.id) - } - documents.push(...batch) - if (rawCount >= pageSize) { - hasFullBatch = true - } - } - return { documents, hasFullBatch } -} - -/** - * Ask for one page of every DDO the node holds, in the form the backend in front of us - * actually accepts. - * - * On Elasticsearch the query object is spread straight into the request body, so `q` - a URI - * search parameter, not a body key - makes the whole request fail with `Unknown key for a - * VALUE_STRING in [q]`. That failure is caught and skipped per schema, which is why the - * republish used to find nothing on the default backend without ever reporting an error. The - * body form of match-everything is `query: { match_all: {} }`; note that `from` there is a - * 1-based *page number*, not a document offset - that layer converts it. - * - * On Typesense `q: '*'` is the documented match-everything wildcard and needs no `query_by`, - * and paging is passed as the second and third arguments rather than inside the query. - * - * Known ceiling, stated so it is not rediscovered as a bug: Elasticsearch refuses a - * `from` + `size` window larger than `index.max_result_window` (10000 by default) with - * "Result window is too large", and the database layer catches that per schema like any other - * failure - so a *single* index holding more than 10000 DDOs re-provides only the first 10000 - * of them, quietly. Lifting that means paging with `search_after` or a point-in-time cursor - * instead of an offset, which is a change to the database layer, not to this job. - */ -function searchDdoPage( - ddoDb: any, - useElasticQuery: boolean, - page: number, - pageSize: number -): Promise { - if (useElasticQuery) { - return ddoDb.search({ query: { match_all: {} }, size: pageSize, from: page }) - } - return ddoDb.search({ q: '*' }, pageSize, page) -} - -/** - * Which query form to send, decided from the shape of the schemas the DDO database carries: an - * Elasticsearch schema names an `index`, a Typesense one a `name`. Read off the schemas rather - * than off the configured database type because the schemas are what the search call is going - * to be issued against, and they are reachable through the public interface. - */ -function usesElasticQuery(ddoDb: any): boolean { - const schemas = typeof ddoDb.getSchemas === 'function' ? ddoDb.getSchemas() : [] - return Array.isArray(schemas) && schemas.length > 0 && schemas[0]?.index !== undefined -} +const BY_ID: SortSpec[] = [{ paths: DDO_FIELDS.id, order: 'asc' }] export async function p2pAnnounceDDOS(node: OceanNode) { try { @@ -133,31 +30,30 @@ export async function p2pAnnounceDDOS(node: OceanNode) { return } const ddoDb = db.ddo - const useElasticQuery = usesElasticQuery(ddoDb) let advertised = 0 let seen = 0 let failed = 0 let cacheFailed = 0 - let page = 1 - // Walk until the store is exhausted. Fetching a single page and stopping meant everything - // past the first page - and, on Typesense, everything outside the first collection - was - // never re-provided, so those DIDs stayed unreachable until something else advertised them. - while (page <= MAX_REPUBLISH_PAGES) { - const result = await searchDdoPage( - ddoDb, - useElasticQuery, - page, - REPUBLISH_PAGE_SIZE - ) - const { documents, hasFullBatch } = extractPage(result, REPUBLISH_PAGE_SIZE) + // Walk all DDOs, every state, in the order of the id. Each page starts after the last + // id of the page before, not at an offset: Elasticsearch refuses an offset past its + // result window (10,000 documents). + let lastId: string | undefined + for (;;) { + const afterLast: Condition[] = lastId + ? [{ paths: DDO_FIELDS.id, range: { gt: lastId } }] + : [] + const { results } = await ddoDb.find(afterLast, BY_ID, { + offset: 0, + limit: REPUBLISH_PAGE_SIZE + }) + const documents = results.filter((document: any) => document && document.id) if (documents.length === 0) { break } seen += documents.length - P2P_LOGGER.debug( - `Will republish cid for ${documents.length} documents (page ${page})` - ) + lastId = documents[documents.length - 1].id + P2P_LOGGER.debug(`Will republish cid for ${documents.length} documents`) // bounded concurrency and a real await. `hits.forEach(hit => advertiseString(...))` // returned before a single provide had completed, so nothing capped the fan-out and the // caller could not know when (or whether) the republish finished. @@ -199,16 +95,9 @@ export async function p2pAnnounceDDOS(node: OceanNode) { }) ) ) - if (!hasFullBatch) { + if (results.length < REPUBLISH_PAGE_SIZE) { break } - page++ - } - - if (page > MAX_REPUBLISH_PAGES) { - P2P_LOGGER.warn( - `Stopped republishing after ${MAX_REPUBLISH_PAGES} pages; the store still reports more documents` - ) } if (seen === 0) { diff --git a/src/utils/cronjobs/scheduleCronJobs.ts b/src/utils/cronjobs/scheduleCronJobs.ts index d338baa7f..477eeacb7 100644 --- a/src/utils/cronjobs/scheduleCronJobs.ts +++ b/src/utils/cronjobs/scheduleCronJobs.ts @@ -15,6 +15,9 @@ import { sleep } from '../util.js' // (we can put smaller interval for testing purposes) const REPUBLISH_INTERVAL_HOURS = 1000 * 60 * 60 * 4 // 4 hours const CLEAN_EXPIRED_JOBS_THROTTLE_KEY = 'cron:cleanExpiredC2DJobs' +const CLEAN_EXPIRED_AUTH_TOKENS_THROTTLE_KEY = 'cron:cleanExpiredAuthTokens' +// daily, at a quiet off-the-hour minute +const CLEAN_EXPIRED_AUTH_TOKENS_SCHEDULE = '17 2 * * *' export async function scheduleCronJobs(node: OceanNode) { await sleep(2000) // wait for 2 seconds to ensure the node is fully initialized @@ -28,6 +31,11 @@ export async function scheduleCronJobs(node: OceanNode) { } catch (e) { OCEAN_NODE_LOGGER.error(`Error when deleting expired c2d jobs: ${e.message}`) } + try { + scheduleAuthTokenCleanup(await node.getDatabase()) + } catch (e) { + OCEAN_NODE_LOGGER.error(`Error when deleting expired auth tokens: ${e.message}`) + } try { scheduleNodeMetricsJobs(node, await node.getDatabase()) } catch (e) { @@ -114,3 +122,25 @@ function scheduleCleanExpiredC2DJobs(dbconn: Database | null) { ) } } + +function scheduleAuthTokenCleanup(dbconn: Database | null) { + if (dbconn && dbconn.authToken) { + scheduleCron(CLEAN_EXPIRED_AUTH_TOKENS_SCHEDULE, async () => { + try { + const deleted = await dbconn.authToken.deleteExpiredTokens(Date.now()) + OCEAN_NODE_LOGGER.resetThrottle(CLEAN_EXPIRED_AUTH_TOKENS_THROTTLE_KEY) + OCEAN_NODE_LOGGER.debug(`${deleted} expired auth tokens deleted`) + } catch (err) { + OCEAN_NODE_LOGGER.throttled( + CLEAN_EXPIRED_AUTH_TOKENS_THROTTLE_KEY, + LOG_LEVELS_STR.LEVEL_ERROR, + `Error deleting expired auth tokens: ${err.message}` + ) + } + }) + } else { + OCEAN_NODE_LOGGER.warn( + 'Auth token CronJob: Database connection not established or auth token instance not available (skipped).' + ) + } +} diff --git a/src/utils/database.ts b/src/utils/database.ts index cad5d5745..28a398b80 100644 --- a/src/utils/database.ts +++ b/src/utils/database.ts @@ -1,34 +1,18 @@ import { OceanNodeDBConfig } from '../@types/OceanNode.js' -import { DB_TYPES } from './constants.js' -import { URLUtils } from './url.js' -export function hasValidDBConfiguration(configuration: OceanNodeDBConfig): boolean { - if (!configuration || !configuration.dbType) { - return false +// What is wrong with the connection values, or undefined. SQLite needs none; a database +// server needs all three. The URL is not checked: the driver of DB_TYPE connects to it. +export function dbConfigurationProblem({ + dbType, + url, + username, + password +}: OceanNodeDBConfig): string | undefined { + if (dbType === 'sqlite') { + return undefined } - const hasValidUrl = configuration.url && URLUtils.isValidUrl(configuration.url) - const hasValidDbType = [DB_TYPES.ELASTIC_SEARCH, DB_TYPES.TYPESENSE].includes( - configuration.dbType - ) - - if (!hasValidUrl || !hasValidDbType) { - return false - } - - // For Elasticsearch, username and password are required - if (configuration.dbType === DB_TYPES.ELASTIC_SEARCH) { - return !!(configuration.username && configuration.password) - } - - return true -} - -// we can use this to check if DB connection is available -export async function isReachableConnection(url: string): Promise { - try { - await fetch(url) - return true - } catch (error) { - return false + if (!url || !username || !password) { + return `DB_URL, DB_USERNAME and DB_PASSWORD are required when DB_TYPE is ${dbType}` } + return undefined } diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts index 8cc109259..e10e74570 100644 --- a/src/utils/logging/Logger.ts +++ b/src/utils/logging/Logger.ts @@ -151,7 +151,7 @@ export class CustomOceanNodesTransport extends Transport { this.emit('logged', info) }) - // Prevent feedback loop: writing a log to the DB triggers Typesense API calls, + // Prevent feedback loop: writing a log to the DB triggers database calls, // which generate debug logs, which would trigger more DB writes, and so on. if (this.isLogging) { callback() diff --git a/typesense-compose.yml b/typesense-compose.yml deleted file mode 100644 index 6d244c47d..000000000 --- a/typesense-compose.yml +++ /dev/null @@ -1,9 +0,0 @@ -services: - typesense: - image: typesense/typesense:0.25.1 - restart: on-failure - ports: - - '8108:8108' - volumes: - - ./typesense-data:/data - command: '--data-dir /data --api-key=xyz'