From cd634aea8afc80b073cc40882848aba1f126330c Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 19 Nov 2026 08:20:07 +0200 Subject: [PATCH 1/7] fix flaky `credentials` integration test: --- src/components/Indexer/index.ts | 103 ++++++++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 10 deletions(-) diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index d98210f63..93039db03 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -60,7 +60,6 @@ let INDEXING_QUEUE: ReindexTask[] = [] const JOBS_QUEUE: JobStatus[] = [] const MAX_CRAWL_RETRIES = 10 -let numCrawlAttempts = 0 /** * OceanIndexer - Multi-chain blockchain event indexer @@ -94,6 +93,25 @@ export class OceanIndexer { private MIN_REQUIRED_VERSION = '4.2.0' private isDbConnected: boolean = true private reconnectTimer: NodeJS.Timeout | null = null + // Per-instance crawl-retry counter. It used to be a module-level global shared by every + // OceanIndexer, so one instance's retries could push another over MAX_CRAWL_RETRIES (and a + // reconnect in one instance reset the counter for all). Each indexer now owns its own. + private numCrawlAttempts = 0 + // Bound ES-connection handlers, retained so stop() can detach them. They were registered as + // anonymous listeners on the module-level ES_CONNECTION_EVENTS emitter and never removed, so + // every OceanIndexer created in the shared-process test suite leaked a pair that kept reacting + // (and could restart a torn-down indexer with its stale config) long after the instance was gone. + private onDbConnectionLost: () => Promise + private onDbConnectionRestored: () => void + // Set true by stop()/stopAllChainIndexers(), false by startAllChainIndexers(). It closes a + // start/stop race: startChainIndexer can sleep for seconds inside retryCrawlerWithDelay while + // the blockchain connection comes up, and during that window the ChainIndexer does not yet + // exist in `this.indexers`, so a concurrent stop finds nothing to cancel. Without this guard + // the indexer would resurrect itself after the sleep and keep crawling past the stop — which, + // in the shared-process integration suite, left a stale indexer (whose config lacked the + // suite's AUTHORIZED_PUBLISHERS gate) crawling chain 8996 and indexing assets a later suite + // expects to be rejected (the flaky credentials "should NOT allow to index" failures). + private stopRequested: boolean = false constructor( _db: Database, @@ -118,7 +136,7 @@ export class OceanIndexer { * CONNECTION_RESTORED → debounce restart by 5 s so rapid LOST/RESTORED cycles are a single restart. */ private setupDbConnectionListeners(): void { - ES_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_LOST, async () => { + this.onDbConnectionLost = async () => { if (this.reconnectTimer) { clearTimeout(this.reconnectTimer) this.reconnectTimer = null @@ -133,9 +151,9 @@ export class OceanIndexer { 'Database connection lost - stopping all chain indexers until DB is back' ) await this.stopAllChainIndexers() - }) + } - ES_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_RESTORED, () => { + this.onDbConnectionRestored = () => { if (this.isDbConnected) { return } @@ -151,14 +169,41 @@ export class OceanIndexer { } this.isDbConnected = true - numCrawlAttempts = 0 + this.numCrawlAttempts = 0 INDEXER_LOGGER.info( 'Database connection stable - reinitialising DB and restarting all chain indexers' ) await this.startAllChainIndexers() }, 5000) - }) + } + + ES_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_LOST, this.onDbConnectionLost) + ES_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_RESTORED, this.onDbConnectionRestored) + } + + /** + * Detach this instance's ES-connection listeners and cancel any pending reconnect timer. + * Called from stop() so a torn-down indexer stops reacting to DB connection events (otherwise + * every OceanIndexer ever created would keep listening on the shared emitter). + */ + private teardownDbConnectionListeners(): void { + if (this.onDbConnectionLost) { + ES_CONNECTION_EVENTS.removeListener( + DB_EVENTS.CONNECTION_LOST, + this.onDbConnectionLost + ) + } + if (this.onDbConnectionRestored) { + ES_CONNECTION_EVENTS.removeListener( + DB_EVENTS.CONNECTION_RESTORED, + this.onDbConnectionRestored + ) + } + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer) + this.reconnectTimer = null + } } public getSupportedNetworks(): RPCS { @@ -189,6 +234,10 @@ export class OceanIndexer { // stops all indexers public async stopAllChainIndexers(): Promise { + // Signal any in-flight startChainIndexer (e.g. one still sleeping in retryCrawlerWithDelay + // before its ChainIndexer is registered) to abort instead of starting to crawl after we + // return. + this.stopRequested = true const stopPromises: Promise[] = [] for (const chainID of this.supportedChains) { const promise = this.stopChainIndexer(Number(chainID)) @@ -244,10 +293,18 @@ export class OceanIndexer { retryInterval / 1000 } secs....` ) - numCrawlAttempts++ - if (numCrawlAttempts <= MAX_CRAWL_RETRIES) { + this.numCrawlAttempts++ + if (this.numCrawlAttempts <= MAX_CRAWL_RETRIES) { // delay the next call await sleep(retryInterval) + // Abort the retry chain if a stop was requested while we slept, so a stopped indexer + // doesn't keep retrying (and eventually start crawling) after it was torn down. + if (this.stopRequested) { + INDEXER_LOGGER.debug( + 'Stop requested during connection retry; aborting crawler start' + ) + return false + } // recursively call the same func return this.retryCrawlerWithDelay(blockchain, retryInterval) } else { @@ -264,7 +321,17 @@ export class OceanIndexer { } // starts indexing for a specific chain - public async startChainIndexer(chainID: number): Promise { + public async startChainIndexer( + chainID: number, + // When this is an explicit, standalone start (admin START_THREAD, resetCrawling) rather than + // part of a startAllChainIndexers() batch, clear any lingering stop request first: the caller + // is explicitly asking to (re)start this chain. The batch path passes false and manages the + // flag itself, so a teardown that lands mid-batch is still honoured (see `stopRequested`). + resetStopRequested: boolean = true + ): Promise { + if (resetStopRequested) { + this.stopRequested = false + } // If an indexer is already running, stop it first const existingIndexer = this.indexers.get(chainID) if (existingIndexer && existingIndexer.isIndexing()) { @@ -296,6 +363,16 @@ export class OceanIndexer { return null } + // A stop may have been requested while retryCrawlerWithDelay was waiting for the connection. + // Do NOT start crawling in that case — otherwise this indexer outlives the stop and keeps + // polling the chain with a now-stale config (see `stopRequested`). + if (this.stopRequested) { + INDEXER_LOGGER.debug( + `Stop requested while connecting; not starting indexer for chain ${chainID}` + ) + return null + } + // Create new ChainIndexer instance const indexer = new ChainIndexer( blockchain, @@ -323,6 +400,9 @@ export class OceanIndexer { // so two indexers never run concurrently against the same DB/chain. await OceanNode.awaitPendingTeardown() + // Clear any stop requested by a previous lifecycle so this start is honoured. + this.stopRequested = false + await this.checkAndTriggerReindexing() // Setup event listeners for all chains (they all use the same event emitter) @@ -332,7 +412,9 @@ export class OceanIndexer { let count = 0 for (const network of this.supportedChains) { const chainId = parseInt(network) - const indexer = await this.startChainIndexer(chainId) + // Don't let the per-chain start clear stopRequested: the batch owns the flag (cleared + // once above) so a stop arriving mid-batch still aborts the chains still coming up. + const indexer = await this.startChainIndexer(chainId, false) if (indexer) { count++ } @@ -601,6 +683,7 @@ export class OceanIndexer { public async stop(): Promise { await this.stopAllChainIndexers() + this.teardownDbConnectionListeners() INDEXER_CRAWLING_EVENT_EMITTER.removeAllListeners() INDEXER_DDO_EVENT_EMITTER.removeAllListeners() } From 47b01f82b03d9e072aed1879e23640892e6c2e64 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 19 Nov 2026 11:25:53 +0200 Subject: [PATCH 2/7] fix review comments --- src/components/Indexer/index.ts | 97 ++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 33 deletions(-) diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index 93039db03..255307831 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -103,15 +103,21 @@ export class OceanIndexer { // (and could restart a torn-down indexer with its stale config) long after the instance was gone. private onDbConnectionLost: () => Promise private onDbConnectionRestored: () => void - // Set true by stop()/stopAllChainIndexers(), false by startAllChainIndexers(). It closes a + // Generation-based cancellation for in-flight starts. Each start operation — a + // startAllChainIndexers() batch, or a standalone startChainIndexer() — takes the next value of + // startGeneration; stopAllChainIndexers() records the latest generation in canceledGeneration. + // A start whose generation is <= canceledGeneration aborts instead of crawling. This closes a // start/stop race: startChainIndexer can sleep for seconds inside retryCrawlerWithDelay while // the blockchain connection comes up, and during that window the ChainIndexer does not yet - // exist in `this.indexers`, so a concurrent stop finds nothing to cancel. Without this guard - // the indexer would resurrect itself after the sleep and keep crawling past the stop — which, - // in the shared-process integration suite, left a stale indexer (whose config lacked the + // exist in `this.indexers`, so a concurrent stop finds nothing to cancel. A single boolean flag + // was not enough: a later standalone start would reset it and revive an already-stopped start. + // In the shared-process integration suite this left a stale indexer (whose config lacked the // suite's AUTHORIZED_PUBLISHERS gate) crawling chain 8996 and indexing assets a later suite - // expects to be rejected (the flaky credentials "should NOT allow to index" failures). - private stopRequested: boolean = false + // expects to be rejected (the flaky credentials "should NOT allow to index" failures). Tying + // cancellation to a per-start generation means a stop only cancels the starts already issued, + // and a fresh start (allocating a higher generation) is never revived or blocked by it. + private startGeneration = 0 + private canceledGeneration = 0 constructor( _db: Database, @@ -234,10 +240,11 @@ export class OceanIndexer { // stops all indexers public async stopAllChainIndexers(): Promise { - // Signal any in-flight startChainIndexer (e.g. one still sleeping in retryCrawlerWithDelay - // before its ChainIndexer is registered) to abort instead of starting to crawl after we - // return. - this.stopRequested = true + // Cancel any in-flight start (e.g. one still sleeping in retryCrawlerWithDelay before its + // ChainIndexer is registered): mark every generation issued so far as canceled, so those + // starts abort instead of beginning to crawl after we return. A later start allocates a + // higher generation and is unaffected. + this.canceledGeneration = this.startGeneration const stopPromises: Promise[] = [] for (const chainID of this.supportedChains) { const promise = this.stopChainIndexer(Number(chainID)) @@ -261,6 +268,11 @@ export class OceanIndexer { } } + // A start (batch or standalone) is canceled when its generation has been superseded by a stop. + private isStartCanceled(generation: number): boolean { + return this.canceledGeneration >= generation + } + // it does not start crawling until the network connectin is ready async startCrawler(blockchain: Blockchain): Promise { if ((await blockchain.isNetworkReady()).ready) { @@ -271,8 +283,12 @@ export class OceanIndexer { async retryCrawlerWithDelay( blockchain: Blockchain, - interval: number = 5000 // in milliseconds, default 2 secs + interval: number = 5000, // in milliseconds + generation?: number ): Promise { + // The start generation this retry belongs to. If that start is stopped while we sleep, abort + // instead of eventually crawling (see `startGeneration`/`canceledGeneration`). + const gen = generation ?? this.startGeneration try { const retryInterval = Math.max(blockchain.getKnownRPCs().length * 3000, interval) // give 2 secs per each one const result = await this.startCrawler(blockchain) @@ -297,16 +313,16 @@ export class OceanIndexer { if (this.numCrawlAttempts <= MAX_CRAWL_RETRIES) { // delay the next call await sleep(retryInterval) - // Abort the retry chain if a stop was requested while we slept, so a stopped indexer + // Abort the retry chain if this start was stopped while we slept, so a stopped indexer // doesn't keep retrying (and eventually start crawling) after it was torn down. - if (this.stopRequested) { + if (this.isStartCanceled(gen)) { INDEXER_LOGGER.debug( 'Stop requested during connection retry; aborting crawler start' ) return false } - // recursively call the same func - return this.retryCrawlerWithDelay(blockchain, retryInterval) + // recursively call the same func, preserving this start's generation + return this.retryCrawlerWithDelay(blockchain, retryInterval, gen) } else { INDEXER_LOGGER.error( `Giving up start crawling after ${MAX_CRAWL_RETRIES} retries.` @@ -323,15 +339,12 @@ export class OceanIndexer { // starts indexing for a specific chain public async startChainIndexer( chainID: number, - // When this is an explicit, standalone start (admin START_THREAD, resetCrawling) rather than - // part of a startAllChainIndexers() batch, clear any lingering stop request first: the caller - // is explicitly asking to (re)start this chain. The batch path passes false and manages the - // flag itself, so a teardown that lands mid-batch is still honoured (see `stopRequested`). - resetStopRequested: boolean = true + // A startAllChainIndexers() batch passes its shared generation so a single stop cancels the + // whole batch. A standalone start (admin START_THREAD, resetCrawling) omits it and takes a + // fresh generation, so a prior stop never blocks a later explicit (re)start of this chain. + batchGeneration?: number ): Promise { - if (resetStopRequested) { - this.stopRequested = false - } + const generation = batchGeneration ?? ++this.startGeneration // If an indexer is already running, stop it first const existingIndexer = this.indexers.get(chainID) if (existingIndexer && existingIndexer.isIndexing()) { @@ -355,7 +368,7 @@ export class OceanIndexer { INDEXER_LOGGER.error(`Unable to get Blockchain instance for chain: ${chainID}`) return null } - const canStartIndexer = await this.retryCrawlerWithDelay(blockchain) + const canStartIndexer = await this.retryCrawlerWithDelay(blockchain, 5000, generation) if (!canStartIndexer) { INDEXER_LOGGER.error( `Cannot start indexer for chain ${chainID}. Check DB and RPC connections!` @@ -365,8 +378,8 @@ export class OceanIndexer { // A stop may have been requested while retryCrawlerWithDelay was waiting for the connection. // Do NOT start crawling in that case — otherwise this indexer outlives the stop and keeps - // polling the chain with a now-stale config (see `stopRequested`). - if (this.stopRequested) { + // polling the chain with a now-stale config (see `startGeneration`/`canceledGeneration`). + if (this.isStartCanceled(generation)) { INDEXER_LOGGER.debug( `Stop requested while connecting; not starting indexer for chain ${chainID}` ) @@ -395,15 +408,29 @@ export class OceanIndexer { // Start all chain indexers public async startAllChainIndexers(): Promise { + // Allocate this batch's generation BEFORE any await, so a stop that lands while we are still + // awaiting teardown/reindex is recorded against this generation and aborts the batch + // (see `startGeneration`/`canceledGeneration`). + const generation = ++this.startGeneration + // Close the overlap at its source: wait for a previous OceanNode instance's teardown // (its indexer stop + provider destroy) to finish before this indexer starts crawling, // so two indexers never run concurrently against the same DB/chain. await OceanNode.awaitPendingTeardown() - - // Clear any stop requested by a previous lifecycle so this start is honoured. - this.stopRequested = false + if (this.isStartCanceled(generation)) { + INDEXER_LOGGER.debug( + 'Stop requested during startup (awaiting teardown); aborting start of all chain indexers' + ) + return false + } await this.checkAndTriggerReindexing() + if (this.isStartCanceled(generation)) { + INDEXER_LOGGER.debug( + 'Stop requested during startup (reindex check); aborting start of all chain indexers' + ) + return false + } // Setup event listeners for all chains (they all use the same event emitter) this.setupEventListeners() @@ -412,9 +439,9 @@ export class OceanIndexer { let count = 0 for (const network of this.supportedChains) { const chainId = parseInt(network) - // Don't let the per-chain start clear stopRequested: the batch owns the flag (cleared - // once above) so a stop arriving mid-batch still aborts the chains still coming up. - const indexer = await this.startChainIndexer(chainId, false) + // Share this batch's generation across all chains, so a single stop cancels the whole + // batch — including chains still coming up. + const indexer = await this.startChainIndexer(chainId, generation) if (indexer) { count++ } @@ -682,8 +709,12 @@ export class OceanIndexer { } public async stop(): Promise { - await this.stopAllChainIndexers() + // Detach DB listeners and cancel any pending reconnect BEFORE stopping, so a + // CONNECTION_RESTORED arriving mid-stop cannot schedule a restart. stopAllChainIndexers() + // then marks in-flight starts (including an already-fired reconnect that is mid-startup) as + // canceled, so none can resurrect an indexer after shutdown. this.teardownDbConnectionListeners() + await this.stopAllChainIndexers() INDEXER_CRAWLING_EVENT_EMITTER.removeAllListeners() INDEXER_DDO_EVENT_EMITTER.removeAllListeners() } From b2b805313b32c3fb0db4643035e28142775cd750 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 19 Nov 2026 12:30:03 +0200 Subject: [PATCH 3/7] more fixing --- src/components/Indexer/index.ts | 74 ++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index 255307831..0f8f52d72 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -118,6 +118,11 @@ export class OceanIndexer { // and a fresh start (allocating a higher generation) is never revived or blocked by it. private startGeneration = 0 private canceledGeneration = 0 + // In-flight startChainIndexer() calls. stopAllChainIndexers() awaits these before tearing down + // registered indexers, so a start that is between ChainIndexer.start() (which launches the + // background crawl loop) and registration in `this.indexers` is waited for and stopped, instead + // of slipping through that gap and crawling on after the stop returns. + private pendingStarts: Set> = new Set() constructor( _db: Database, @@ -245,6 +250,15 @@ export class OceanIndexer { // starts abort instead of beginning to crawl after we return. A later start allocates a // higher generation and is unaffected. this.canceledGeneration = this.startGeneration + + // Wait for in-flight starts to settle before tearing down registered indexers. Each now sees + // its generation canceled and either aborts before crawling or stops the indexer it just + // started (see startChainIndexer), so none survives in the gap between ChainIndexer.start() + // and registration. Snapshot the set because entries remove themselves as they settle. + if (this.pendingStarts.size > 0) { + await Promise.allSettled([...this.pendingStarts]) + } + const stopPromises: Promise[] = [] for (const chainID of this.supportedChains) { const promise = this.stopChainIndexer(Number(chainID)) @@ -273,6 +287,22 @@ export class OceanIndexer { return this.canceledGeneration >= generation } + // Sleep up to `ms`, but wake early (returning true) as soon as the given start generation is + // canceled. Keeps a stop during a connection retry from blocking shutdown for the full retry + // interval (stopAllChainIndexers() awaits in-flight starts). + private async sleepUnlessCanceled(ms: number, generation: number): Promise { + const sliceMs = 200 + let waited = 0 + while (waited < ms) { + if (this.isStartCanceled(generation)) { + return true + } + await sleep(Math.min(sliceMs, ms - waited)) + waited += sliceMs + } + return this.isStartCanceled(generation) + } + // it does not start crawling until the network connectin is ready async startCrawler(blockchain: Blockchain): Promise { if ((await blockchain.isNetworkReady()).ready) { @@ -311,11 +341,10 @@ export class OceanIndexer { ) this.numCrawlAttempts++ if (this.numCrawlAttempts <= MAX_CRAWL_RETRIES) { - // delay the next call - await sleep(retryInterval) - // Abort the retry chain if this start was stopped while we slept, so a stopped indexer - // doesn't keep retrying (and eventually start crawling) after it was torn down. - if (this.isStartCanceled(gen)) { + // delay the next call, but abort the retry chain promptly if this start was stopped + // while we waited, so a stopped indexer doesn't keep retrying (and eventually start + // crawling) after it was torn down. + if (await this.sleepUnlessCanceled(retryInterval, gen)) { INDEXER_LOGGER.debug( 'Stop requested during connection retry; aborting crawler start' ) @@ -345,6 +374,21 @@ export class OceanIndexer { batchGeneration?: number ): Promise { const generation = batchGeneration ?? ++this.startGeneration + // Register this start as in-flight so a concurrent stopAllChainIndexers() waits for it to + // settle (and stops any indexer it creates) rather than racing past it. + const startPromise = this.doStartChainIndexer(chainID, generation) + this.pendingStarts.add(startPromise) + try { + return await startPromise + } finally { + this.pendingStarts.delete(startPromise) + } + } + + private async doStartChainIndexer( + chainID: number, + generation: number + ): Promise { // If an indexer is already running, stop it first const existingIndexer = this.indexers.get(chainID) if (existingIndexer && existingIndexer.isIndexing()) { @@ -402,6 +446,18 @@ export class OceanIndexer { // Start indexing (runs in background, doesn't block) await indexer.start() + // ChainIndexer.start() launches the background crawl loop but does NOT register the indexer in + // `this.indexers`, so until the set() below a concurrent stopAllChainIndexers() cannot reach + // it. If a stop landed during start(), stop this freshly started indexer here instead of + // leaving it crawling past the stop (see `startGeneration`/`canceledGeneration`). + if (this.isStartCanceled(generation)) { + INDEXER_LOGGER.debug( + `Stop requested while starting; stopping freshly started indexer for chain ${chainID}` + ) + await indexer.stop() + return null + } + this.indexers.set(chainID, indexer) return indexer } @@ -438,6 +494,14 @@ export class OceanIndexer { // Start all indexers - they will run concurrently via async/await let count = 0 for (const network of this.supportedChains) { + // Stop the batch promptly if a stop landed mid-loop, instead of running a network/DB probe + // per remaining chain only for each start to self-abort on the canceled generation. + if (this.isStartCanceled(generation)) { + INDEXER_LOGGER.debug( + 'Stop requested during startup (starting chains); aborting remaining chain indexers' + ) + break + } const chainId = parseInt(network) // Share this batch's generation across all chains, so a single stop cancels the whole // batch — including chains still coming up. From 4aa85003307d57aea6e0ff994322647b3b36902e Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 19 Nov 2026 12:49:20 +0200 Subject: [PATCH 4/7] more fixes --- src/components/Indexer/index.ts | 42 +++++++++++++++++++++++---------- src/utils/database.ts | 12 +++++++--- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index 0f8f52d72..72687b7e5 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -251,23 +251,39 @@ export class OceanIndexer { // higher generation and is unaffected. this.canceledGeneration = this.startGeneration - // Wait for in-flight starts to settle before tearing down registered indexers. Each now sees - // its generation canceled and either aborts before crawling or stops the indexer it just - // started (see startChainIndexer), so none survives in the gap between ChainIndexer.start() - // and registration. Snapshot the set because entries remove themselves as they settle. - if (this.pendingStarts.size > 0) { - await Promise.allSettled([...this.pendingStarts]) - } - + // Snapshot the indexers registered right now. We tear down exactly these instances — NOT + // whatever the map holds after the join below. A start issued after this point has a higher, + // non-canceled generation and may register during the join; tearing it down here would break + // the contract that a later start is unaffected by this stop. + const toStop: Array<[number, ChainIndexer]> = [...this.indexers.entries()] + + // Stop the already-registered crawlers FIRST — before joining in-flight starts below — so a + // slow or hung start (e.g. one awaiting the DB-reachability probe) cannot keep live crawlers + // indexing during shutdown. Confirm the map still holds this exact instance and evict it + // atomically (no await between the check and the delete) before stopping it, so a newer + // indexer at the same chain is left running. const stopPromises: Promise[] = [] - for (const chainID of this.supportedChains) { - const promise = this.stopChainIndexer(Number(chainID)) - if (promise) { - stopPromises.push(promise) + for (const [chainID, indexer] of toStop) { + if (this.indexers.get(chainID) !== indexer) { + continue } + this.indexers.delete(chainID) + INDEXER_LOGGER.debug(`Stopping indexer for chain ${chainID}`) + stopPromises.push(indexer.stop()) } await Promise.allSettled(stopPromises) - return stopPromises.length === this.supportedChains.length + + // Then wait for in-flight starts to settle. A canceled start aborts before registering, or + // stops the indexer it just started (see doStartChainIndexer); a later, non-canceled start + // registers and is intentionally left running. isReachableConnection() is time-bounded, so + // this join cannot block shutdown indefinitely on a start stuck in its connection probe. + if (this.pendingStarts.size > 0) { + await Promise.allSettled([...this.pendingStarts]) + } + + // Preserve the historical always-true result (the previous loop pushed one promise per chain + // unconditionally); no caller inspects it. + return true } // stops indexing for a specific chain diff --git a/src/utils/database.ts b/src/utils/database.ts index cad5d5745..d17925f7c 100644 --- a/src/utils/database.ts +++ b/src/utils/database.ts @@ -23,10 +23,16 @@ export function hasValidDBConfiguration(configuration: OceanNodeDBConfig): boole return true } -// we can use this to check if DB connection is available -export async function isReachableConnection(url: string): Promise { +// we can use this to check if DB connection is available. The fetch is time-bounded (AbortSignal) +// so a URL that accepts the connection but never responds is reported as unreachable after +// `timeoutMs` instead of hanging the caller indefinitely (startup check, indexer connection retry, +// and the shutdown join that waits on in-flight starts all rely on this returning). +export async function isReachableConnection( + url: string, + timeoutMs: number = 5000 +): Promise { try { - await fetch(url) + await fetch(url, { signal: AbortSignal.timeout(timeoutMs) }) return true } catch (error) { return false From 826eadb8dff820c3608a675362c642c41353e863 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 19 Nov 2026 13:24:26 +0200 Subject: [PATCH 5/7] fix indexer lifecycle --- src/components/Indexer/ChainIndexer.ts | 151 +++++++++--- src/components/Indexer/index.ts | 222 +++++++++++++----- .../core/admin/IndexingThreadHandler.ts | 38 ++- src/utils/database.ts | 12 +- 4 files changed, 316 insertions(+), 107 deletions(-) diff --git a/src/components/Indexer/ChainIndexer.ts b/src/components/Indexer/ChainIndexer.ts index 3017662e7..9b7cd1658 100644 --- a/src/components/Indexer/ChainIndexer.ts +++ b/src/components/Indexer/ChainIndexer.ts @@ -22,6 +22,17 @@ import { Database } from '../database/index.js' // consecutive loop failures after which a recovery is logged at info and the throttle reset const FAILURE_ESCALATION_THRESHOLD = 5 +// Thrown by raceStop() when a stop is requested while an await is still pending, so the indexing +// loop can unwind promptly instead of blocking the stop() handshake on an unbounded RPC/DB call +// (which would otherwise deadlock: stop() busy-waits on isRunning, which the loop only clears when +// it exits). +class IndexerStoppedError extends Error { + constructor(label: string) { + super(`ChainIndexer stopped during ${label}`) + this.name = 'IndexerStoppedError' + } +} + export interface ReindexTask { txId: string chainId: number @@ -43,6 +54,9 @@ export class ChainIndexer { private blockchain: Blockchain private db: Database private consecutiveFailures: number = 0 + // Aborted by stop() so any await wrapped in raceStop() rejects promptly instead of blocking the + // shutdown handshake on an unbounded RPC/DB call. Re-created in start() for each run. + private abortController: AbortController = new AbortController() constructor( blockchain: Blockchain, @@ -71,6 +85,8 @@ export class ChainIndexer { } this.stopSignal = false + // Fresh controller for this run (a previous run's controller may already be aborted). + this.abortController = new AbortController() this.isRunning = true // Start crawling but DON'T await - let it run in background @@ -87,6 +103,9 @@ export class ChainIndexer { */ async stop(): Promise { this.stopSignal = true + // Unblock any await the loop is currently parked on (raceStop), so the busy-wait below ends + // promptly instead of hanging until an unbounded RPC/DB call returns. + this.abortController.abort() INDEXER_LOGGER.debug( `Stopping indexer for chain ${this.blockchain.getSupportedChain()}, waiting for graceful shutdown...` ) @@ -99,6 +118,33 @@ export class ChainIndexer { INDEXER_LOGGER.info(`Chain ${this.blockchain.getSupportedChain()}: indexer stopped`) } + /** + * Race an await against this indexer's stop. If stop() aborts while `operation` is still pending, + * reject with IndexerStoppedError so indexLoop() unwinds immediately instead of blocking stop() + * on an unbounded RPC/DB call. The underlying operation keeps running but its result is ignored; + * its eventual settlement is consumed here so a late rejection is not left unhandled. + */ + private raceStop(operation: Promise, label: string): Promise { + const { signal } = this.abortController + if (signal.aborted) { + return Promise.reject(new IndexerStoppedError(label)) + } + return new Promise((resolve, reject) => { + const onAbort = () => reject(new IndexerStoppedError(label)) + signal.addEventListener('abort', onAbort, { once: true }) + operation.then( + (value) => { + signal.removeEventListener('abort', onAbort) + resolve(value) + }, + (err) => { + signal.removeEventListener('abort', onAbort) + reject(err) + } + ) + }) + } + /** * Check if the indexer is currently running */ @@ -170,8 +216,21 @@ export class ChainIndexer { `Chain ${this.blockchain.getSupportedChain()}: start block ${crawlingStartBlock} (deploy ${contractDeploymentBlock}, configured startBlock ${this.rpcDetails.startBlock})` ) - let provider = await this.blockchain.getProvider() - let signer = await this.blockchain.getSigner() + let provider: FallbackProvider + let signer: Signer + try { + provider = await this.raceStop(this.blockchain.getProvider(), 'getProvider') + signer = await this.raceStop(this.blockchain.getSigner(), 'getSigner') + } catch (error) { + if (error instanceof IndexerStoppedError || this.stopSignal) { + INDEXER_LOGGER.debug( + `Chain ${this.blockchain.getSupportedChain()}: stop requested before indexing loop started` + ) + this.isRunning = false + return + } + throw error + } const interval = getCrawlingInterval() let chunkSize = this.rpcDetails.chunkSize || 1 let successfulRetrievalCount = 0 @@ -188,11 +247,17 @@ export class ChainIndexer { // Self-heal: if the provider was destroyed (overlapping teardown), getProvider() // recreates it and getSigner() rebuilds against it. When healthy, both return the // cached instances at negligible cost. - provider = await this.blockchain.getProvider() - signer = await this.blockchain.getSigner() + provider = await this.raceStop(this.blockchain.getProvider(), 'getProvider') + signer = await this.raceStop(this.blockchain.getSigner(), 'getSigner') - const lastIndexedBlock = await this.getLastIndexedBlock() - const networkHeight = await getNetworkHeight(provider) + const lastIndexedBlock = await this.raceStop( + this.getLastIndexedBlock(), + 'getLastIndexedBlock' + ) + const networkHeight = await this.raceStop( + getNetworkHeight(provider), + 'getNetworkHeight' + ) const startBlock = lastIndexedBlock && lastIndexedBlock > crawlingStartBlock ? lastIndexedBlock @@ -222,15 +287,20 @@ export class ChainIndexer { let chunkEvents: Log[] = [] try { - chunkEvents = await retrieveChunkEvents( - signer, - provider, - this.blockchain.getSupportedChain(), - startBlock, - blocksToProcess + chunkEvents = await this.raceStop( + retrieveChunkEvents( + signer, + provider, + this.blockchain.getSupportedChain(), + startBlock, + blocksToProcess + ), + 'retrieveChunkEvents' ) successfulRetrievalCount++ } catch (error) { + // Let a stop unwind the loop instead of being treated as a getLogs failure. + if (error instanceof IndexerStoppedError) throw error // Rethrow so the outer catch retries this range instead of advancing past it. if (isProviderError(error)) throw error chunkSize = Math.floor(chunkSize / 2) < 1 ? 1 : Math.floor(chunkSize / 2) @@ -246,23 +316,29 @@ export class ChainIndexer { } try { - const processedBlocks = await processBlocks( - chunkEvents, - signer, - provider, - this.blockchain.getSupportedChain(), - startBlock, - blocksToProcess, - this.config + const processedBlocks = await this.raceStop( + processBlocks( + chunkEvents, + signer, + provider, + this.blockchain.getSupportedChain(), + startBlock, + blocksToProcess, + this.config + ), + 'processBlocks' ) INDEXER_LOGGER.debug( `Processed ${Object.keys(processedBlocks.foundEvents).length} event types from ${chunkEvents.length} logs` ) - currentBlock = await this.updateLastIndexedBlockNumber( - processedBlocks.lastBlock, - lastIndexedBlock + currentBlock = await this.raceStop( + this.updateLastIndexedBlockNumber( + processedBlocks.lastBlock, + lastIndexedBlock + ), + 'updateLastIndexedBlockNumber' ) // Can't update currentBlock to processedBlocks.lastBlock if DB action failed @@ -284,6 +360,8 @@ export class ChainIndexer { ) } } catch (error) { + // A stop requested mid-processing must unwind the loop, not be retried as a failure. + if (error instanceof IndexerStoppedError) throw error iterationFailed = true this.logLoopFailure( `Chain ${this.blockchain.getSupportedChain()}: processing blocks from ${startBlock + 1} failed (${errorSummary(error)}), retrying` @@ -298,12 +376,23 @@ export class ChainIndexer { } // Process reindex queue - await this.processReindexQueue(provider, signer) + await this.raceStop( + this.processReindexQueue(provider, signer), + 'processReindexQueue' + ) - // Handle chain reindex command - if (this.reindexBlock !== null && !lockProcessing) { - const networkHeight = await getNetworkHeight(provider) - const result = await this.reindexChain(currentBlock, networkHeight) + // Handle chain reindex command. (The old `&& !lockProcessing` guard was always false + // here — lockProcessing is set true at the top of this block — so a triggered full-chain + // reindex never ran. reindexChain() clears reindexBlock on every path, so this cannot loop.) + if (this.reindexBlock !== null) { + const networkHeight = await this.raceStop( + getNetworkHeight(provider), + 'getNetworkHeight' + ) + const result = await this.raceStop( + this.reindexChain(currentBlock, networkHeight), + 'reindexChain' + ) this.eventEmitter.emit(INDEXER_CRAWLING_EVENTS.REINDEX_CHAIN, { result, @@ -312,6 +401,12 @@ export class ChainIndexer { } if (!iterationFailed) this.logLoopRecovered() } catch (error) { + // Stop requested mid-iteration (raceStop) or stopSignal set: exit the loop promptly + // instead of logging a failure and sleeping/retrying. The finally still clears the lock, + // then the while-condition sees stopSignal and the loop ends (isRunning cleared below). + if (error instanceof IndexerStoppedError || this.stopSignal) { + break + } this.logLoopFailure( `Error in indexing loop for chain ${this.blockchain.getSupportedChain()}: ${errorSummary(error)}` ) diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index 72687b7e5..9b391bbee 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -93,10 +93,19 @@ export class OceanIndexer { private MIN_REQUIRED_VERSION = '4.2.0' private isDbConnected: boolean = true private reconnectTimer: NodeJS.Timeout | null = null - // Per-instance crawl-retry counter. It used to be a module-level global shared by every - // OceanIndexer, so one instance's retries could push another over MAX_CRAWL_RETRIES (and a - // reconnect in one instance reset the counter for all). Each indexer now owns its own. - private numCrawlAttempts = 0 + // This instance's crawling-event handlers, retained so stop() detaches exactly them instead of + // calling removeAllListeners() on the shared INDEXER_CRAWLING_EVENT_EMITTER (which would also + // drop other OceanIndexers' handlers). setupEventListeners() also detaches the previous set + // before re-registering, so repeated startAllChainIndexers() calls (e.g. on DB reconnect) don't + // stack duplicate handlers on the shared emitter. + private crawlingEventHandlers: Array<{ + event: string + handler: (payload: any) => void + }> = [] + // Pending addJob() status-promotion timers, cleared in stop() so a torn-down indexer's 2 s timer + // doesn't fire after shutdown and mutate the shared JOBS_QUEUE (which leaks across the + // shared-process test suite). + private jobTimers: Set = new Set() // Bound ES-connection handlers, retained so stop() can detach them. They were registered as // anonymous listeners on the module-level ES_CONNECTION_EVENTS emitter and never removed, so // every OceanIndexer created in the shared-process test suite leaked a pair that kept reacting @@ -123,6 +132,14 @@ export class OceanIndexer { // background crawl loop) and registration in `this.indexers` is waited for and stopped, instead // of slipping through that gap and crawling on after the stop returns. private pendingStarts: Set> = new Set() + // One AbortController per in-flight start, tagged with the start's generation. stopAllChainIndexers() + // aborts the controllers of canceled generations so a start blocked on the DB-reachability probe + // (an otherwise-unbounded fetch) unblocks promptly — without which the shutdown join could hang on + // it. When no stop occurs a probe is never aborted, so its result is unchanged. + private startProbeControllers: Set<{ + generation: number + controller: AbortController + }> = new Set() constructor( _db: Database, @@ -136,6 +153,9 @@ export class OceanIndexer { this.supportedChains = Object.keys(this.networks) clearEventProcessorCache() INDEXING_QUEUE = [] + // JOBS_QUEUE is a module-level global that was never reset, so jobs from a previous OceanIndexer + // leaked into this one (the shared-process test suite). Reset it like INDEXING_QUEUE above. + JOBS_QUEUE.length = 0 this.setupDbConnectionListeners() this.startAllChainIndexers() } @@ -180,7 +200,6 @@ export class OceanIndexer { } this.isDbConnected = true - this.numCrawlAttempts = 0 INDEXER_LOGGER.info( 'Database connection stable - reinitialising DB and restarting all chain indexers' ) @@ -251,6 +270,15 @@ export class OceanIndexer { // higher generation and is unaffected. this.canceledGeneration = this.startGeneration + // Abort the DB-reachability probe of every canceled in-flight start, so one blocked on an + // unresponsive probe fails fast (isReachableConnection → false → the start gives up) instead of + // hanging the join below. Non-canceled starts (higher generation) keep their probe untouched. + for (const probe of this.startProbeControllers) { + if (probe.generation <= this.canceledGeneration) { + probe.controller.abort() + } + } + // Snapshot the indexers registered right now. We tear down exactly these instances — NOT // whatever the map holds after the join below. A start issued after this point has a higher, // non-canceled generation and may register during the join; tearing it down here would break @@ -275,7 +303,7 @@ export class OceanIndexer { // Then wait for in-flight starts to settle. A canceled start aborts before registering, or // stops the indexer it just started (see doStartChainIndexer); a later, non-canceled start - // registers and is intentionally left running. isReachableConnection() is time-bounded, so + // registers and is intentionally left running. We aborted canceled starts' DB probes above, so // this join cannot block shutdown indefinitely on a start stuck in its connection probe. if (this.pendingStarts.size > 0) { await Promise.allSettled([...this.pendingStarts]) @@ -330,7 +358,13 @@ export class OceanIndexer { async retryCrawlerWithDelay( blockchain: Blockchain, interval: number = 5000, // in milliseconds - generation?: number + generation?: number, + // Aborted by stopAllChainIndexers() when this start's generation is canceled, so the + // DB-reachability probe below cannot hang the start (and the shutdown join) indefinitely. + probeSignal?: AbortSignal, + // Retry count carried through the recursion (was a shared instance/global field, so one chain's + // retries could push another past MAX_CRAWL_RETRIES). Local to this start's retry chain now. + attempt: number = 0 ): Promise { // The start generation this retry belongs to. If that start is stopped while we sleep, abort // instead of eventually crawling (see `startGeneration`/`canceledGeneration`). @@ -339,7 +373,10 @@ export class OceanIndexer { const retryInterval = Math.max(blockchain.getKnownRPCs().length * 3000, interval) // give 2 secs per each one const result = await this.startCrawler(blockchain) const dbActive = this.getDatabase() - if (!dbActive || !(await isReachableConnection(dbActive.getConfig().url))) { + if ( + !dbActive || + !(await isReachableConnection(dbActive.getConfig().url, probeSignal)) + ) { INDEXER_LOGGER.error(`Giving up start crawling. DB is not online!`) return false } @@ -355,8 +392,8 @@ export class OceanIndexer { retryInterval / 1000 } secs....` ) - this.numCrawlAttempts++ - if (this.numCrawlAttempts <= MAX_CRAWL_RETRIES) { + const nextAttempt = attempt + 1 + if (nextAttempt <= MAX_CRAWL_RETRIES) { // delay the next call, but abort the retry chain promptly if this start was stopped // while we waited, so a stopped indexer doesn't keep retrying (and eventually start // crawling) after it was torn down. @@ -366,8 +403,14 @@ export class OceanIndexer { ) return false } - // recursively call the same func, preserving this start's generation - return this.retryCrawlerWithDelay(blockchain, retryInterval, gen) + // recursively call the same func, preserving this start's generation and probe signal + return this.retryCrawlerWithDelay( + blockchain, + retryInterval, + gen, + probeSignal, + nextAttempt + ) } else { INDEXER_LOGGER.error( `Giving up start crawling after ${MAX_CRAWL_RETRIES} retries.` @@ -390,21 +433,43 @@ export class OceanIndexer { batchGeneration?: number ): Promise { const generation = batchGeneration ?? ++this.startGeneration - // Register this start as in-flight so a concurrent stopAllChainIndexers() waits for it to - // settle (and stops any indexer it creates) rather than racing past it. - const startPromise = this.doStartChainIndexer(chainID, generation) + // Register this start as in-flight (plus a probe AbortController tagged with its generation) so + // a concurrent stopAllChainIndexers() can both wait for it to settle and abort its DB probe, + // rather than racing past it or hanging on it. + const probe = { generation, controller: new AbortController() } + this.startProbeControllers.add(probe) + const startPromise = this.doStartChainIndexer( + chainID, + generation, + probe.controller.signal + ) this.pendingStarts.add(startPromise) try { return await startPromise } finally { this.pendingStarts.delete(startPromise) + this.startProbeControllers.delete(probe) } } private async doStartChainIndexer( chainID: number, - generation: number + generation: number, + probeSignal: AbortSignal ): Promise { + // A standalone start (admin START_THREAD, resetCrawling) does not go through + // startAllChainIndexers()'s teardown wait, so wait here too: a previous OceanNode instance's + // teardown (its indexer stop + provider destroy) must finish before this indexer starts + // crawling, or two indexers run concurrently against the same DB/chain. awaitPendingTeardown() + // is a no-op when nothing is pending, so the batch path (which already awaited it) is unaffected. + await OceanNode.awaitPendingTeardown() + if (this.isStartCanceled(generation)) { + INDEXER_LOGGER.debug( + `Stop requested during startup (awaiting teardown); not starting indexer for chain ${chainID}` + ) + return null + } + // If an indexer is already running, stop it first const existingIndexer = this.indexers.get(chainID) if (existingIndexer && existingIndexer.isIndexing()) { @@ -428,7 +493,12 @@ export class OceanIndexer { INDEXER_LOGGER.error(`Unable to get Blockchain instance for chain: ${chainID}`) return null } - const canStartIndexer = await this.retryCrawlerWithDelay(blockchain, 5000, generation) + const canStartIndexer = await this.retryCrawlerWithDelay( + blockchain, + 5000, + generation, + probeSignal + ) if (!canStartIndexer) { INDEXER_LOGGER.error( `Cannot start indexer for chain ${chainID}. Check DB and RPC connections!` @@ -530,7 +600,27 @@ export class OceanIndexer { return count === this.supportedChains.length } + // Register a crawling-event handler and track it so stop() can detach exactly this instance's set. + private onCrawling(event: string, handler: (payload: any) => void): void { + this.crawlingEventHandlers.push({ event, handler }) + INDEXER_CRAWLING_EVENT_EMITTER.on(event, handler) + } + + // Detach every crawling-event handler this instance registered (NOT removeAllListeners(), which + // would also drop handlers belonging to other OceanIndexers sharing the module-level emitter). + private teardownEventListeners(): void { + for (const { event, handler } of this.crawlingEventHandlers) { + INDEXER_CRAWLING_EVENT_EMITTER.removeListener(event, handler) + } + this.crawlingEventHandlers = [] + } + private setupEventListeners() { + // Detach any previously registered handlers first: setupEventListeners() runs on every + // startAllChainIndexers() (including DB-reconnect restarts), so without this each restart would + // stack another full set of handlers on the shared emitter (duplicate DDO emissions, leaks). + this.teardownEventListeners() + // Listen to metadata events from any chain indexer const metadataEvents = [ EVENTS.METADATA_CREATED, @@ -546,7 +636,7 @@ export class OceanIndexer { ] metadataEvents.forEach((eventType) => { - INDEXER_CRAWLING_EVENT_EMITTER.on(eventType, async (event: any) => { + this.onCrawling(eventType, async (event: any) => { try { if (!event.data) { INDEXER_LOGGER.debug( @@ -568,49 +658,40 @@ export class OceanIndexer { }) // Listen to reindex queue pop events - INDEXER_CRAWLING_EVENT_EMITTER.on( - INDEXER_CRAWLING_EVENTS.REINDEX_QUEUE_POP, - (event: any) => { - try { - INDEXING_QUEUE = INDEXING_QUEUE.filter( - (task) => task.txId !== event.txId && task.chainId !== event.chainId - ) - this.updateJobStatus( - PROTOCOL_COMMANDS.REINDEX_TX, - create256Hash([event.chainId, event.txId].join('')), - CommandStatus.SUCCESS - ) - } catch (err) { - INDEXER_LOGGER.error(`Reindex queue pop handler failed: ${err?.message ?? err}`) - } + this.onCrawling(INDEXER_CRAWLING_EVENTS.REINDEX_QUEUE_POP, (event: any) => { + try { + INDEXING_QUEUE = INDEXING_QUEUE.filter( + (task) => task.txId !== event.txId && task.chainId !== event.chainId + ) + this.updateJobStatus( + PROTOCOL_COMMANDS.REINDEX_TX, + create256Hash([event.chainId, event.txId].join('')), + CommandStatus.SUCCESS + ) + } catch (err) { + INDEXER_LOGGER.error(`Reindex queue pop handler failed: ${err?.message ?? err}`) } - ) + }) // Listen to reindex chain events - INDEXER_CRAWLING_EVENT_EMITTER.on( - INDEXER_CRAWLING_EVENTS.REINDEX_CHAIN, - (event: any) => { - try { - this.updateJobStatus( - PROTOCOL_COMMANDS.REINDEX_CHAIN, - create256Hash([event.chainId].join('')), - event.result ? CommandStatus.SUCCESS : CommandStatus.FAILURE - ) - } catch (err) { - INDEXER_LOGGER.error(`Reindex chain handler failed: ${err?.message ?? err}`) - } + this.onCrawling(INDEXER_CRAWLING_EVENTS.REINDEX_CHAIN, (event: any) => { + try { + this.updateJobStatus( + PROTOCOL_COMMANDS.REINDEX_CHAIN, + create256Hash([event.chainId].join('')), + event.result ? CommandStatus.SUCCESS : CommandStatus.FAILURE + ) + } catch (err) { + INDEXER_LOGGER.error(`Reindex chain handler failed: ${err?.message ?? err}`) } - ) + }) // Listen to crawling started events - INDEXER_CRAWLING_EVENT_EMITTER.on( - INDEXER_CRAWLING_EVENTS.CRAWLING_STARTED, - (event: any) => { - INDEXER_LOGGER.info( - `Chain ${event.chainId}: crawling from block ${event.startBlock}` - ) - } - ) + this.onCrawling(INDEXER_CRAWLING_EVENTS.CRAWLING_STARTED, (event: any) => { + INDEXER_LOGGER.info( + `Chain ${event.chainId}: crawling from block ${event.startBlock}` + ) + }) } public addReindexTask(reindexTask: ReindexTask): JobStatus | null { @@ -666,16 +747,18 @@ export class OceanIndexer { public getJobsPool(jobId?: string): JobStatus[] { if (jobId) { - const pos = -1 const result = this.filterJobs(jobId) - // if it finished, then we can remove it from the list of jobs + // if it finished, then we can remove it from the list of jobs. Use the job's actual index + // (result.position) returned by filterJobs — a hardcoded -1 spliced the LAST job instead, + // removing the wrong entry whenever the finished job was not at the end of the queue. if ( + result.position >= 0 && result.jobsResult.length === 1 && [CommandStatus.FAILURE, CommandStatus.SUCCESS].includes( result.jobsResult[0].status ) ) { - JOBS_QUEUE.splice(pos, 1) + JOBS_QUEUE.splice(result.position, 1) } return result.jobsResult } @@ -685,7 +768,10 @@ export class OceanIndexer { // when we add a new job, we change the status from DELIVERED to PENDING if still running after a couple secs public addJob(jobInfo: JobStatus) { JOBS_QUEUE.push(jobInfo) - setTimeout(() => { + // Track the timer so stop() can clear it; otherwise a torn-down OceanIndexer leaves a pending + // 2 s timer that fires after shutdown and mutates the shared JOBS_QUEUE. + const timer = setTimeout(() => { + this.jobTimers.delete(timer) const result = this.filterJobs(jobInfo.jobId) if ( result.jobsResult.length === 1 && @@ -694,6 +780,15 @@ export class OceanIndexer { this.updateJobStatus(jobInfo.command, jobInfo.hash, CommandStatus.PENDING) } }, 2000) + this.jobTimers.add(timer) + } + + // Cancel any pending addJob() status-promotion timers (called from stop()). + private clearJobTimers(): void { + for (const timer of this.jobTimers) { + clearTimeout(timer) + } + this.jobTimers.clear() } // filter jobs by job id, return the position of the job on the queue as well @@ -795,7 +890,12 @@ export class OceanIndexer { // canceled, so none can resurrect an indexer after shutdown. this.teardownDbConnectionListeners() await this.stopAllChainIndexers() - INDEXER_CRAWLING_EVENT_EMITTER.removeAllListeners() - INDEXER_DDO_EVENT_EMITTER.removeAllListeners() + // Detach only THIS instance's crawling-event handlers and cancel its pending job timers. Do + // NOT removeAllListeners() on the shared emitters: INDEXER_CRAWLING_EVENT_EMITTER is shared by + // any future OceanIndexer, and INDEXER_DDO_EVENT_EMITTER has external subscribers (the P2P + // node's METADATA_CREATED handler, test utils) that a blanket removeAllListeners() would + // silently unsubscribe. + this.teardownEventListeners() + this.clearJobTimers() } } diff --git a/src/components/core/admin/IndexingThreadHandler.ts b/src/components/core/admin/IndexingThreadHandler.ts index 8c6eddb30..11d1b40ff 100644 --- a/src/components/core/admin/IndexingThreadHandler.ts +++ b/src/components/core/admin/IndexingThreadHandler.ts @@ -38,26 +38,40 @@ export class IndexingThreadHandler extends AdminCommandHandler { } CORE_LOGGER.info(`Admin ${task.action} for chain ${task.chainId ?? 'all'}`) if (task.action === IndexingCommand.START_THREAD) { - const output = task.chainId - ? indexer.startChainIndexer(task.chainId) - : indexer.startAllChainIndexers() + // Await the start: these are async operations (they can sleep for seconds while the chain + // connection comes up). Without the await `output` was always a truthy Promise, so the + // handler reported HTTP 200 before the indexer had actually started (or failed). A per-chain + // start resolves to the ChainIndexer or null; a start-all resolves to a boolean. + const success = task.chainId + ? (await indexer.startChainIndexer(task.chainId)) !== null + : (await indexer.startAllChainIndexers()) === true return { status: { - httpStatus: output ? 200 : 400, - error: output ? null : 'Unable to start indexing thread(s)!' + httpStatus: success ? 200 : 400, + error: success ? null : 'Unable to start indexing thread(s)!' }, - stream: output ? new ReadableString('OK') : null + stream: success ? new ReadableString('OK') : null } } else if (task.action === IndexingCommand.STOP_THREAD) { - const output = task.chainId - ? indexer.stopChainIndexer(task.chainId) - : indexer.stopAllChainIndexers() + // Await the stop for the same reason. stopChainIndexer resolves to void (success unless it + // throws); stopAllChainIndexers resolves to a boolean. + let success = true + try { + if (task.chainId) { + await indexer.stopChainIndexer(task.chainId) + } else { + success = (await indexer.stopAllChainIndexers()) === true + } + } catch (error) { + CORE_LOGGER.error(`Unable to stop indexing thread(s): ${error?.message ?? error}`) + success = false + } return { status: { - httpStatus: output ? 200 : 400, - error: output ? null : 'Unable to stop indexing thread(s)!' + httpStatus: success ? 200 : 400, + error: success ? null : 'Unable to stop indexing thread(s)!' }, - stream: output ? new ReadableString('OK') : null + stream: success ? new ReadableString('OK') : null } } } diff --git a/src/utils/database.ts b/src/utils/database.ts index d17925f7c..54f3cb269 100644 --- a/src/utils/database.ts +++ b/src/utils/database.ts @@ -23,16 +23,16 @@ export function hasValidDBConfiguration(configuration: OceanNodeDBConfig): boole return true } -// we can use this to check if DB connection is available. The fetch is time-bounded (AbortSignal) -// so a URL that accepts the connection but never responds is reported as unreachable after -// `timeoutMs` instead of hanging the caller indefinitely (startup check, indexer connection retry, -// and the shutdown join that waits on in-flight starts all rely on this returning). +// we can use this to check if DB connection is available. An optional AbortSignal lets the caller +// cancel a probe that would otherwise hang on a URL that accepts the connection but never responds +// (the indexer aborts it when the owning start is stopped). With no signal the behaviour is +// unchanged: the probe resolves true/false purely on the fetch outcome, however long that takes. export async function isReachableConnection( url: string, - timeoutMs: number = 5000 + signal?: AbortSignal ): Promise { try { - await fetch(url, { signal: AbortSignal.timeout(timeoutMs) }) + await fetch(url, signal ? { signal } : undefined) return true } catch (error) { return false From f2ae0c49a278014aa323bd76a33fa357a28e15e5 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 19 Nov 2026 15:32:49 +0200 Subject: [PATCH 6/7] fix --- src/OceanNode.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/OceanNode.ts b/src/OceanNode.ts index 5bdb0f4d0..48dd45617 100644 --- a/src/OceanNode.ts +++ b/src/OceanNode.ts @@ -186,7 +186,14 @@ export class OceanNode { public addIndexer(_indexer: OceanIndexer) { const previous = this.indexer this.indexer = _indexer - if (previous) { + // Only stop a genuinely DIFFERENT previous indexer. The constructor's `private indexer` + // parameter property has already assigned this.indexer to the passed instance before the + // constructor body calls addIndexer(indexer), so here `previous === _indexer`. Without this + // guard, construction immediately stops the just-created indexer — and because stop() now + // cancels in-flight starts by generation, that permanently aborts its startup crawl (the node + // then never indexes anything). A real replacement (tests, a new OceanNode instance) passes a + // different instance and still stops the old one. + if (previous && previous !== _indexer) { previous.stop().catch((err: unknown) => { OCEAN_NODE_LOGGER.error( `Failed to stop replaced indexer: ${err instanceof Error ? err.message : String(err)}` From 3fd40b89d5e4f91b2f4829207608a30368c6f733 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 19 Nov 2026 15:56:06 +0200 Subject: [PATCH 7/7] fix review comments --- src/components/Indexer/ChainIndexer.ts | 105 +++++++++++++------------ src/components/Indexer/index.ts | 18 +++-- 2 files changed, 67 insertions(+), 56 deletions(-) diff --git a/src/components/Indexer/ChainIndexer.ts b/src/components/Indexer/ChainIndexer.ts index 9b7cd1658..66751ceb2 100644 --- a/src/components/Indexer/ChainIndexer.ts +++ b/src/components/Indexer/ChainIndexer.ts @@ -236,7 +236,6 @@ export class ChainIndexer { let successfulRetrievalCount = 0 let lockProcessing = false let startedCrawling = false - let currentBlock: number while (!this.stopSignal) { if (!lockProcessing) { @@ -316,36 +315,30 @@ export class ChainIndexer { } try { - const processedBlocks = await this.raceStop( - processBlocks( - chunkEvents, - signer, - provider, - this.blockchain.getSupportedChain(), - startBlock, - blocksToProcess, - this.config - ), - 'processBlocks' + // NOT raced against stop: processBlocks and updateLastIndexedBlockNumber WRITE to the + // DB (stored DDOs/orders + the indexer cursor). They must run to completion before the + // loop can exit, otherwise an orphaned write from a stopped run could race a + // replacement indexer for the same chain. stop() waits for them (as it did before the + // raceStop() wrapper existed); only the hang-prone RPC/DB reads above stay raced. + const processedBlocks = await processBlocks( + chunkEvents, + signer, + provider, + this.blockchain.getSupportedChain(), + startBlock, + blocksToProcess, + this.config ) INDEXER_LOGGER.debug( `Processed ${Object.keys(processedBlocks.foundEvents).length} event types from ${chunkEvents.length} logs` ) - currentBlock = await this.raceStop( - this.updateLastIndexedBlockNumber( - processedBlocks.lastBlock, - lastIndexedBlock - ), - 'updateLastIndexedBlockNumber' + await this.updateLastIndexedBlockNumber( + processedBlocks.lastBlock, + lastIndexedBlock ) - // Can't update currentBlock to processedBlocks.lastBlock if DB action failed - if (currentBlock < 0 && lastIndexedBlock !== null) { - currentBlock = lastIndexedBlock - } - this.emitNewlyIndexedAssets(processedBlocks.foundEvents) // Revert to original chunk size after 3 successful retrievals @@ -360,8 +353,6 @@ export class ChainIndexer { ) } } catch (error) { - // A stop requested mid-processing must unwind the loop, not be retried as a failure. - if (error instanceof IndexerStoppedError) throw error iterationFailed = true this.logLoopFailure( `Chain ${this.blockchain.getSupportedChain()}: processing blocks from ${startBlock + 1} failed (${errorSummary(error)}), retrying` @@ -375,11 +366,9 @@ export class ChainIndexer { await sleep(interval) } - // Process reindex queue - await this.raceStop( - this.processReindexQueue(provider, signer), - 'processReindexQueue' - ) + // Process reindex queue. NOT raced: it writes to the DB and emits queue-pop events, so it + // must finish before the loop exits rather than being orphaned past a stop. + await this.processReindexQueue(provider, signer) // Handle chain reindex command. (The old `&& !lockProcessing` guard was always false // here — lockProcessing is set true at the top of this block — so a triggered full-chain @@ -389,10 +378,10 @@ export class ChainIndexer { getNetworkHeight(provider), 'getNetworkHeight' ) - const result = await this.raceStop( - this.reindexChain(currentBlock, networkHeight), - 'reindexChain' - ) + // reindexChain WRITES to the DB (resets the cursor, purges assets). NOT raced, so it + // can't be orphaned mid-mutation past a stop. It reads the current cursor itself for + // rollback, so no block argument is passed. + const result = await this.reindexChain(networkHeight) this.eventEmitter.emit(INDEXER_CRAWLING_EVENTS.REINDEX_CHAIN, { result, @@ -512,33 +501,49 @@ export class ChainIndexer { /** * Perform a full chain reindex */ - private async reindexChain( - currentBlock: number, - networkHeight: number - ): Promise { - if (this.reindexBlock > networkHeight) { + private async reindexChain(networkHeight: number): Promise { + const targetBlock = this.reindexBlock + if (targetBlock > networkHeight) { INDEXER_LOGGER.warn( - `Invalid reindex block! ${this.reindexBlock} is bigger than network height: ${networkHeight}. Continue indexing normally...` + `Invalid reindex block! ${targetBlock} is bigger than network height: ${networkHeight}. Continue indexing normally...` ) this.reindexBlock = null return false } - const block = await this.updateLastIndexedBlockNumber(this.reindexBlock) - if (block !== -1) { - this.reindexBlock = null - const res = await this.deleteAllAssetsFromChain() - if (res === -1) { - await this.updateLastIndexedBlockNumber(currentBlock) - } - return true - } else { + // Capture the cursor as it stands NOW, before updateLastIndexedBlockNumber overwrites it below, + // so a failed asset purge can be rolled back to the real pre-reindex value instead of a stale + // caller-supplied one. getLastIndexedBlock() returns null when there is no stored cursor OR the + // read failed (it logs and swallows the error); either way we fall back to targetBlock on + // rollback so we never write undefined. + const previousCursor = await this.getLastIndexedBlock() + + const block = await this.updateLastIndexedBlockNumber(targetBlock) + if (block === -1) { INDEXER_LOGGER.error( - `Chain ${this.blockchain.getSupportedChain()}: reindex from block ${this.reindexBlock} dropped, the last indexed block could not be reset. Continue indexing normally...` + `Chain ${this.blockchain.getSupportedChain()}: reindex from block ${targetBlock} dropped, the last indexed block could not be reset. Continue indexing normally...` ) this.reindexBlock = null return false } + + this.reindexBlock = null + const res = await this.deleteAllAssetsFromChain() + if (res === -1) { + // Purge failed: restore the pre-reindex cursor so we don't leave it reset to the deployment + // block while the old assets are still present. Never restore to undefined. + const restoreTo = isDefined(previousCursor) ? previousCursor : targetBlock + const restored = await this.updateLastIndexedBlockNumber(restoreTo) + if (restored === -1) { + INDEXER_LOGGER.error( + `Chain ${this.blockchain.getSupportedChain()}: failed to restore last indexed block to ${restoreTo} after a failed reindex asset purge` + ) + } + return false + } + + // Success — including when zero assets were deleted (res === 0). + return true } /** diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index 9b391bbee..51875f7f4 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -270,6 +270,11 @@ export class OceanIndexer { // higher generation and is unaffected. this.canceledGeneration = this.startGeneration + // Snapshot the in-flight starts at entry, BEFORE any await below. Every start pending right now + // has a generation <= canceledGeneration (just set), so all are canceled; we join exactly these + // and never a later, non-canceled start that registers while we await the crawler stops. + const startsToJoin = [...this.pendingStarts] + // Abort the DB-reachability probe of every canceled in-flight start, so one blocked on an // unresponsive probe fails fast (isReachableConnection → false → the start gives up) instead of // hanging the join below. Non-canceled starts (higher generation) keep their probe untouched. @@ -301,12 +306,13 @@ export class OceanIndexer { } await Promise.allSettled(stopPromises) - // Then wait for in-flight starts to settle. A canceled start aborts before registering, or - // stops the indexer it just started (see doStartChainIndexer); a later, non-canceled start - // registers and is intentionally left running. We aborted canceled starts' DB probes above, so - // this join cannot block shutdown indefinitely on a start stuck in its connection probe. - if (this.pendingStarts.size > 0) { - await Promise.allSettled([...this.pendingStarts]) + // Then wait for the starts that were in flight at entry to settle. Each aborts before + // registering, or stops the indexer it just started (see doStartChainIndexer). A later, + // non-canceled start issued during the stops above is deliberately NOT in this snapshot, so it + // registers and keeps running. We aborted canceled starts' DB probes above, so this join cannot + // block shutdown indefinitely on a start stuck in its connection probe. + if (startsToJoin.length > 0) { + await Promise.allSettled(startsToJoin) } // Preserve the historical always-true result (the previous loop pushed one promise per chain