diff --git a/src/OceanNode.ts b/src/OceanNode.ts index 5bdb0f4d0..48dd45617 100644 --- a/src/OceanNode.ts +++ b/src/OceanNode.ts @@ -186,7 +186,14 @@ export class OceanNode { public addIndexer(_indexer: OceanIndexer) { const previous = this.indexer this.indexer = _indexer - if (previous) { + // Only stop a genuinely DIFFERENT previous indexer. The constructor's `private indexer` + // parameter property has already assigned this.indexer to the passed instance before the + // constructor body calls addIndexer(indexer), so here `previous === _indexer`. Without this + // guard, construction immediately stops the just-created indexer — and because stop() now + // cancels in-flight starts by generation, that permanently aborts its startup crawl (the node + // then never indexes anything). A real replacement (tests, a new OceanNode instance) passes a + // different instance and still stops the old one. + if (previous && previous !== _indexer) { previous.stop().catch((err: unknown) => { OCEAN_NODE_LOGGER.error( `Failed to stop replaced indexer: ${err instanceof Error ? err.message : String(err)}` diff --git a/src/components/Indexer/ChainIndexer.ts b/src/components/Indexer/ChainIndexer.ts index 3017662e7..66751ceb2 100644 --- a/src/components/Indexer/ChainIndexer.ts +++ b/src/components/Indexer/ChainIndexer.ts @@ -22,6 +22,17 @@ import { Database } from '../database/index.js' // consecutive loop failures after which a recovery is logged at info and the throttle reset const FAILURE_ESCALATION_THRESHOLD = 5 +// Thrown by raceStop() when a stop is requested while an await is still pending, so the indexing +// loop can unwind promptly instead of blocking the stop() handshake on an unbounded RPC/DB call +// (which would otherwise deadlock: stop() busy-waits on isRunning, which the loop only clears when +// it exits). +class IndexerStoppedError extends Error { + constructor(label: string) { + super(`ChainIndexer stopped during ${label}`) + this.name = 'IndexerStoppedError' + } +} + export interface ReindexTask { txId: string chainId: number @@ -43,6 +54,9 @@ export class ChainIndexer { private blockchain: Blockchain private db: Database private consecutiveFailures: number = 0 + // Aborted by stop() so any await wrapped in raceStop() rejects promptly instead of blocking the + // shutdown handshake on an unbounded RPC/DB call. Re-created in start() for each run. + private abortController: AbortController = new AbortController() constructor( blockchain: Blockchain, @@ -71,6 +85,8 @@ export class ChainIndexer { } this.stopSignal = false + // Fresh controller for this run (a previous run's controller may already be aborted). + this.abortController = new AbortController() this.isRunning = true // Start crawling but DON'T await - let it run in background @@ -87,6 +103,9 @@ export class ChainIndexer { */ async stop(): Promise { this.stopSignal = true + // Unblock any await the loop is currently parked on (raceStop), so the busy-wait below ends + // promptly instead of hanging until an unbounded RPC/DB call returns. + this.abortController.abort() INDEXER_LOGGER.debug( `Stopping indexer for chain ${this.blockchain.getSupportedChain()}, waiting for graceful shutdown...` ) @@ -99,6 +118,33 @@ export class ChainIndexer { INDEXER_LOGGER.info(`Chain ${this.blockchain.getSupportedChain()}: indexer stopped`) } + /** + * Race an await against this indexer's stop. If stop() aborts while `operation` is still pending, + * reject with IndexerStoppedError so indexLoop() unwinds immediately instead of blocking stop() + * on an unbounded RPC/DB call. The underlying operation keeps running but its result is ignored; + * its eventual settlement is consumed here so a late rejection is not left unhandled. + */ + private raceStop(operation: Promise, label: string): Promise { + const { signal } = this.abortController + if (signal.aborted) { + return Promise.reject(new IndexerStoppedError(label)) + } + return new Promise((resolve, reject) => { + const onAbort = () => reject(new IndexerStoppedError(label)) + signal.addEventListener('abort', onAbort, { once: true }) + operation.then( + (value) => { + signal.removeEventListener('abort', onAbort) + resolve(value) + }, + (err) => { + signal.removeEventListener('abort', onAbort) + reject(err) + } + ) + }) + } + /** * Check if the indexer is currently running */ @@ -170,14 +216,26 @@ export class ChainIndexer { `Chain ${this.blockchain.getSupportedChain()}: start block ${crawlingStartBlock} (deploy ${contractDeploymentBlock}, configured startBlock ${this.rpcDetails.startBlock})` ) - let provider = await this.blockchain.getProvider() - let signer = await this.blockchain.getSigner() + let provider: FallbackProvider + let signer: Signer + try { + provider = await this.raceStop(this.blockchain.getProvider(), 'getProvider') + signer = await this.raceStop(this.blockchain.getSigner(), 'getSigner') + } catch (error) { + if (error instanceof IndexerStoppedError || this.stopSignal) { + INDEXER_LOGGER.debug( + `Chain ${this.blockchain.getSupportedChain()}: stop requested before indexing loop started` + ) + this.isRunning = false + return + } + throw error + } const interval = getCrawlingInterval() let chunkSize = this.rpcDetails.chunkSize || 1 let successfulRetrievalCount = 0 let lockProcessing = false let startedCrawling = false - let currentBlock: number while (!this.stopSignal) { if (!lockProcessing) { @@ -188,11 +246,17 @@ export class ChainIndexer { // Self-heal: if the provider was destroyed (overlapping teardown), getProvider() // recreates it and getSigner() rebuilds against it. When healthy, both return the // cached instances at negligible cost. - provider = await this.blockchain.getProvider() - signer = await this.blockchain.getSigner() + provider = await this.raceStop(this.blockchain.getProvider(), 'getProvider') + signer = await this.raceStop(this.blockchain.getSigner(), 'getSigner') - const lastIndexedBlock = await this.getLastIndexedBlock() - const networkHeight = await getNetworkHeight(provider) + const lastIndexedBlock = await this.raceStop( + this.getLastIndexedBlock(), + 'getLastIndexedBlock' + ) + const networkHeight = await this.raceStop( + getNetworkHeight(provider), + 'getNetworkHeight' + ) const startBlock = lastIndexedBlock && lastIndexedBlock > crawlingStartBlock ? lastIndexedBlock @@ -222,15 +286,20 @@ export class ChainIndexer { let chunkEvents: Log[] = [] try { - chunkEvents = await retrieveChunkEvents( - signer, - provider, - this.blockchain.getSupportedChain(), - startBlock, - blocksToProcess + chunkEvents = await this.raceStop( + retrieveChunkEvents( + signer, + provider, + this.blockchain.getSupportedChain(), + startBlock, + blocksToProcess + ), + 'retrieveChunkEvents' ) successfulRetrievalCount++ } catch (error) { + // Let a stop unwind the loop instead of being treated as a getLogs failure. + if (error instanceof IndexerStoppedError) throw error // Rethrow so the outer catch retries this range instead of advancing past it. if (isProviderError(error)) throw error chunkSize = Math.floor(chunkSize / 2) < 1 ? 1 : Math.floor(chunkSize / 2) @@ -246,6 +315,11 @@ export class ChainIndexer { } try { + // NOT raced against stop: processBlocks and updateLastIndexedBlockNumber WRITE to the + // DB (stored DDOs/orders + the indexer cursor). They must run to completion before the + // loop can exit, otherwise an orphaned write from a stopped run could race a + // replacement indexer for the same chain. stop() waits for them (as it did before the + // raceStop() wrapper existed); only the hang-prone RPC/DB reads above stay raced. const processedBlocks = await processBlocks( chunkEvents, signer, @@ -260,16 +334,11 @@ export class ChainIndexer { `Processed ${Object.keys(processedBlocks.foundEvents).length} event types from ${chunkEvents.length} logs` ) - currentBlock = await this.updateLastIndexedBlockNumber( + await this.updateLastIndexedBlockNumber( processedBlocks.lastBlock, lastIndexedBlock ) - // Can't update currentBlock to processedBlocks.lastBlock if DB action failed - if (currentBlock < 0 && lastIndexedBlock !== null) { - currentBlock = lastIndexedBlock - } - this.emitNewlyIndexedAssets(processedBlocks.foundEvents) // Revert to original chunk size after 3 successful retrievals @@ -297,13 +366,22 @@ export class ChainIndexer { await sleep(interval) } - // Process reindex queue + // Process reindex queue. NOT raced: it writes to the DB and emits queue-pop events, so it + // must finish before the loop exits rather than being orphaned past a stop. await this.processReindexQueue(provider, signer) - // Handle chain reindex command - if (this.reindexBlock !== null && !lockProcessing) { - const networkHeight = await getNetworkHeight(provider) - const result = await this.reindexChain(currentBlock, networkHeight) + // Handle chain reindex command. (The old `&& !lockProcessing` guard was always false + // here — lockProcessing is set true at the top of this block — so a triggered full-chain + // reindex never ran. reindexChain() clears reindexBlock on every path, so this cannot loop.) + if (this.reindexBlock !== null) { + const networkHeight = await this.raceStop( + getNetworkHeight(provider), + 'getNetworkHeight' + ) + // reindexChain WRITES to the DB (resets the cursor, purges assets). NOT raced, so it + // can't be orphaned mid-mutation past a stop. It reads the current cursor itself for + // rollback, so no block argument is passed. + const result = await this.reindexChain(networkHeight) this.eventEmitter.emit(INDEXER_CRAWLING_EVENTS.REINDEX_CHAIN, { result, @@ -312,6 +390,12 @@ export class ChainIndexer { } if (!iterationFailed) this.logLoopRecovered() } catch (error) { + // Stop requested mid-iteration (raceStop) or stopSignal set: exit the loop promptly + // instead of logging a failure and sleeping/retrying. The finally still clears the lock, + // then the while-condition sees stopSignal and the loop ends (isRunning cleared below). + if (error instanceof IndexerStoppedError || this.stopSignal) { + break + } this.logLoopFailure( `Error in indexing loop for chain ${this.blockchain.getSupportedChain()}: ${errorSummary(error)}` ) @@ -417,33 +501,49 @@ export class ChainIndexer { /** * Perform a full chain reindex */ - private async reindexChain( - currentBlock: number, - networkHeight: number - ): Promise { - if (this.reindexBlock > networkHeight) { + private async reindexChain(networkHeight: number): Promise { + const targetBlock = this.reindexBlock + if (targetBlock > networkHeight) { INDEXER_LOGGER.warn( - `Invalid reindex block! ${this.reindexBlock} is bigger than network height: ${networkHeight}. Continue indexing normally...` + `Invalid reindex block! ${targetBlock} is bigger than network height: ${networkHeight}. Continue indexing normally...` ) this.reindexBlock = null return false } - const block = await this.updateLastIndexedBlockNumber(this.reindexBlock) - if (block !== -1) { - this.reindexBlock = null - const res = await this.deleteAllAssetsFromChain() - if (res === -1) { - await this.updateLastIndexedBlockNumber(currentBlock) - } - return true - } else { + // Capture the cursor as it stands NOW, before updateLastIndexedBlockNumber overwrites it below, + // so a failed asset purge can be rolled back to the real pre-reindex value instead of a stale + // caller-supplied one. getLastIndexedBlock() returns null when there is no stored cursor OR the + // read failed (it logs and swallows the error); either way we fall back to targetBlock on + // rollback so we never write undefined. + const previousCursor = await this.getLastIndexedBlock() + + const block = await this.updateLastIndexedBlockNumber(targetBlock) + if (block === -1) { INDEXER_LOGGER.error( - `Chain ${this.blockchain.getSupportedChain()}: reindex from block ${this.reindexBlock} dropped, the last indexed block could not be reset. Continue indexing normally...` + `Chain ${this.blockchain.getSupportedChain()}: reindex from block ${targetBlock} dropped, the last indexed block could not be reset. Continue indexing normally...` ) this.reindexBlock = null return false } + + this.reindexBlock = null + const res = await this.deleteAllAssetsFromChain() + if (res === -1) { + // Purge failed: restore the pre-reindex cursor so we don't leave it reset to the deployment + // block while the old assets are still present. Never restore to undefined. + const restoreTo = isDefined(previousCursor) ? previousCursor : targetBlock + const restored = await this.updateLastIndexedBlockNumber(restoreTo) + if (restored === -1) { + INDEXER_LOGGER.error( + `Chain ${this.blockchain.getSupportedChain()}: failed to restore last indexed block to ${restoreTo} after a failed reindex asset purge` + ) + } + return false + } + + // Success — including when zero assets were deleted (res === 0). + return true } /** diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index d98210f63..51875f7f4 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -60,7 +60,6 @@ let INDEXING_QUEUE: ReindexTask[] = [] const JOBS_QUEUE: JobStatus[] = [] const MAX_CRAWL_RETRIES = 10 -let numCrawlAttempts = 0 /** * OceanIndexer - Multi-chain blockchain event indexer @@ -94,6 +93,53 @@ export class OceanIndexer { private MIN_REQUIRED_VERSION = '4.2.0' private isDbConnected: boolean = true private reconnectTimer: NodeJS.Timeout | null = null + // This instance's crawling-event handlers, retained so stop() detaches exactly them instead of + // calling removeAllListeners() on the shared INDEXER_CRAWLING_EVENT_EMITTER (which would also + // drop other OceanIndexers' handlers). setupEventListeners() also detaches the previous set + // before re-registering, so repeated startAllChainIndexers() calls (e.g. on DB reconnect) don't + // stack duplicate handlers on the shared emitter. + private crawlingEventHandlers: Array<{ + event: string + handler: (payload: any) => void + }> = [] + // Pending addJob() status-promotion timers, cleared in stop() so a torn-down indexer's 2 s timer + // doesn't fire after shutdown and mutate the shared JOBS_QUEUE (which leaks across the + // shared-process test suite). + private jobTimers: Set = new Set() + // Bound ES-connection handlers, retained so stop() can detach them. They were registered as + // anonymous listeners on the module-level ES_CONNECTION_EVENTS emitter and never removed, so + // every OceanIndexer created in the shared-process test suite leaked a pair that kept reacting + // (and could restart a torn-down indexer with its stale config) long after the instance was gone. + private onDbConnectionLost: () => Promise + private onDbConnectionRestored: () => void + // Generation-based cancellation for in-flight starts. Each start operation — a + // startAllChainIndexers() batch, or a standalone startChainIndexer() — takes the next value of + // startGeneration; stopAllChainIndexers() records the latest generation in canceledGeneration. + // A start whose generation is <= canceledGeneration aborts instead of crawling. This closes a + // start/stop race: startChainIndexer can sleep for seconds inside retryCrawlerWithDelay while + // the blockchain connection comes up, and during that window the ChainIndexer does not yet + // exist in `this.indexers`, so a concurrent stop finds nothing to cancel. A single boolean flag + // was not enough: a later standalone start would reset it and revive an already-stopped start. + // In the shared-process integration suite this left a stale indexer (whose config lacked the + // suite's AUTHORIZED_PUBLISHERS gate) crawling chain 8996 and indexing assets a later suite + // expects to be rejected (the flaky credentials "should NOT allow to index" failures). Tying + // cancellation to a per-start generation means a stop only cancels the starts already issued, + // and a fresh start (allocating a higher generation) is never revived or blocked by it. + private startGeneration = 0 + private canceledGeneration = 0 + // In-flight startChainIndexer() calls. stopAllChainIndexers() awaits these before tearing down + // registered indexers, so a start that is between ChainIndexer.start() (which launches the + // background crawl loop) and registration in `this.indexers` is waited for and stopped, instead + // of slipping through that gap and crawling on after the stop returns. + private pendingStarts: Set> = new Set() + // One AbortController per in-flight start, tagged with the start's generation. stopAllChainIndexers() + // aborts the controllers of canceled generations so a start blocked on the DB-reachability probe + // (an otherwise-unbounded fetch) unblocks promptly — without which the shutdown join could hang on + // it. When no stop occurs a probe is never aborted, so its result is unchanged. + private startProbeControllers: Set<{ + generation: number + controller: AbortController + }> = new Set() constructor( _db: Database, @@ -107,6 +153,9 @@ export class OceanIndexer { this.supportedChains = Object.keys(this.networks) clearEventProcessorCache() INDEXING_QUEUE = [] + // JOBS_QUEUE is a module-level global that was never reset, so jobs from a previous OceanIndexer + // leaked into this one (the shared-process test suite). Reset it like INDEXING_QUEUE above. + JOBS_QUEUE.length = 0 this.setupDbConnectionListeners() this.startAllChainIndexers() } @@ -118,7 +167,7 @@ export class OceanIndexer { * CONNECTION_RESTORED → debounce restart by 5 s so rapid LOST/RESTORED cycles are a single restart. */ private setupDbConnectionListeners(): void { - ES_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_LOST, async () => { + this.onDbConnectionLost = async () => { if (this.reconnectTimer) { clearTimeout(this.reconnectTimer) this.reconnectTimer = null @@ -133,9 +182,9 @@ export class OceanIndexer { 'Database connection lost - stopping all chain indexers until DB is back' ) await this.stopAllChainIndexers() - }) + } - ES_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_RESTORED, () => { + this.onDbConnectionRestored = () => { if (this.isDbConnected) { return } @@ -151,14 +200,40 @@ export class OceanIndexer { } this.isDbConnected = true - numCrawlAttempts = 0 INDEXER_LOGGER.info( 'Database connection stable - reinitialising DB and restarting all chain indexers' ) await this.startAllChainIndexers() }, 5000) - }) + } + + ES_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_LOST, this.onDbConnectionLost) + ES_CONNECTION_EVENTS.on(DB_EVENTS.CONNECTION_RESTORED, this.onDbConnectionRestored) + } + + /** + * Detach this instance's ES-connection listeners and cancel any pending reconnect timer. + * Called from stop() so a torn-down indexer stops reacting to DB connection events (otherwise + * every OceanIndexer ever created would keep listening on the shared emitter). + */ + private teardownDbConnectionListeners(): void { + if (this.onDbConnectionLost) { + ES_CONNECTION_EVENTS.removeListener( + DB_EVENTS.CONNECTION_LOST, + this.onDbConnectionLost + ) + } + if (this.onDbConnectionRestored) { + ES_CONNECTION_EVENTS.removeListener( + DB_EVENTS.CONNECTION_RESTORED, + this.onDbConnectionRestored + ) + } + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer) + this.reconnectTimer = null + } } public getSupportedNetworks(): RPCS { @@ -189,15 +264,60 @@ export class OceanIndexer { // stops all indexers public async stopAllChainIndexers(): Promise { + // Cancel any in-flight start (e.g. one still sleeping in retryCrawlerWithDelay before its + // ChainIndexer is registered): mark every generation issued so far as canceled, so those + // starts abort instead of beginning to crawl after we return. A later start allocates a + // higher generation and is unaffected. + this.canceledGeneration = this.startGeneration + + // Snapshot the in-flight starts at entry, BEFORE any await below. Every start pending right now + // has a generation <= canceledGeneration (just set), so all are canceled; we join exactly these + // and never a later, non-canceled start that registers while we await the crawler stops. + const startsToJoin = [...this.pendingStarts] + + // Abort the DB-reachability probe of every canceled in-flight start, so one blocked on an + // unresponsive probe fails fast (isReachableConnection → false → the start gives up) instead of + // hanging the join below. Non-canceled starts (higher generation) keep their probe untouched. + for (const probe of this.startProbeControllers) { + if (probe.generation <= this.canceledGeneration) { + probe.controller.abort() + } + } + + // Snapshot the indexers registered right now. We tear down exactly these instances — NOT + // whatever the map holds after the join below. A start issued after this point has a higher, + // non-canceled generation and may register during the join; tearing it down here would break + // the contract that a later start is unaffected by this stop. + const toStop: Array<[number, ChainIndexer]> = [...this.indexers.entries()] + + // Stop the already-registered crawlers FIRST — before joining in-flight starts below — so a + // slow or hung start (e.g. one awaiting the DB-reachability probe) cannot keep live crawlers + // indexing during shutdown. Confirm the map still holds this exact instance and evict it + // atomically (no await between the check and the delete) before stopping it, so a newer + // indexer at the same chain is left running. const stopPromises: Promise[] = [] - for (const chainID of this.supportedChains) { - const promise = this.stopChainIndexer(Number(chainID)) - if (promise) { - stopPromises.push(promise) + for (const [chainID, indexer] of toStop) { + if (this.indexers.get(chainID) !== indexer) { + continue } + this.indexers.delete(chainID) + INDEXER_LOGGER.debug(`Stopping indexer for chain ${chainID}`) + stopPromises.push(indexer.stop()) } await Promise.allSettled(stopPromises) - return stopPromises.length === this.supportedChains.length + + // Then wait for the starts that were in flight at entry to settle. Each aborts before + // registering, or stops the indexer it just started (see doStartChainIndexer). A later, + // non-canceled start issued during the stops above is deliberately NOT in this snapshot, so it + // registers and keeps running. We aborted canceled starts' DB probes above, so this join cannot + // block shutdown indefinitely on a start stuck in its connection probe. + if (startsToJoin.length > 0) { + await Promise.allSettled(startsToJoin) + } + + // Preserve the historical always-true result (the previous loop pushed one promise per chain + // unconditionally); no caller inspects it. + return true } // stops indexing for a specific chain @@ -212,6 +332,27 @@ export class OceanIndexer { } } + // A start (batch or standalone) is canceled when its generation has been superseded by a stop. + private isStartCanceled(generation: number): boolean { + return this.canceledGeneration >= generation + } + + // Sleep up to `ms`, but wake early (returning true) as soon as the given start generation is + // canceled. Keeps a stop during a connection retry from blocking shutdown for the full retry + // interval (stopAllChainIndexers() awaits in-flight starts). + private async sleepUnlessCanceled(ms: number, generation: number): Promise { + const sliceMs = 200 + let waited = 0 + while (waited < ms) { + if (this.isStartCanceled(generation)) { + return true + } + await sleep(Math.min(sliceMs, ms - waited)) + waited += sliceMs + } + return this.isStartCanceled(generation) + } + // it does not start crawling until the network connectin is ready async startCrawler(blockchain: Blockchain): Promise { if ((await blockchain.isNetworkReady()).ready) { @@ -222,13 +363,26 @@ export class OceanIndexer { async retryCrawlerWithDelay( blockchain: Blockchain, - interval: number = 5000 // in milliseconds, default 2 secs + interval: number = 5000, // in milliseconds + generation?: number, + // Aborted by stopAllChainIndexers() when this start's generation is canceled, so the + // DB-reachability probe below cannot hang the start (and the shutdown join) indefinitely. + probeSignal?: AbortSignal, + // Retry count carried through the recursion (was a shared instance/global field, so one chain's + // retries could push another past MAX_CRAWL_RETRIES). Local to this start's retry chain now. + attempt: number = 0 ): Promise { + // The start generation this retry belongs to. If that start is stopped while we sleep, abort + // instead of eventually crawling (see `startGeneration`/`canceledGeneration`). + const gen = generation ?? this.startGeneration try { const retryInterval = Math.max(blockchain.getKnownRPCs().length * 3000, interval) // give 2 secs per each one const result = await this.startCrawler(blockchain) const dbActive = this.getDatabase() - if (!dbActive || !(await isReachableConnection(dbActive.getConfig().url))) { + if ( + !dbActive || + !(await isReachableConnection(dbActive.getConfig().url, probeSignal)) + ) { INDEXER_LOGGER.error(`Giving up start crawling. DB is not online!`) return false } @@ -244,12 +398,25 @@ export class OceanIndexer { retryInterval / 1000 } secs....` ) - numCrawlAttempts++ - if (numCrawlAttempts <= MAX_CRAWL_RETRIES) { - // delay the next call - await sleep(retryInterval) - // recursively call the same func - return this.retryCrawlerWithDelay(blockchain, retryInterval) + const nextAttempt = attempt + 1 + if (nextAttempt <= MAX_CRAWL_RETRIES) { + // delay the next call, but abort the retry chain promptly if this start was stopped + // while we waited, so a stopped indexer doesn't keep retrying (and eventually start + // crawling) after it was torn down. + if (await this.sleepUnlessCanceled(retryInterval, gen)) { + INDEXER_LOGGER.debug( + 'Stop requested during connection retry; aborting crawler start' + ) + return false + } + // recursively call the same func, preserving this start's generation and probe signal + return this.retryCrawlerWithDelay( + blockchain, + retryInterval, + gen, + probeSignal, + nextAttempt + ) } else { INDEXER_LOGGER.error( `Giving up start crawling after ${MAX_CRAWL_RETRIES} retries.` @@ -264,7 +431,51 @@ export class OceanIndexer { } // starts indexing for a specific chain - public async startChainIndexer(chainID: number): Promise { + public async startChainIndexer( + chainID: number, + // A startAllChainIndexers() batch passes its shared generation so a single stop cancels the + // whole batch. A standalone start (admin START_THREAD, resetCrawling) omits it and takes a + // fresh generation, so a prior stop never blocks a later explicit (re)start of this chain. + batchGeneration?: number + ): Promise { + const generation = batchGeneration ?? ++this.startGeneration + // Register this start as in-flight (plus a probe AbortController tagged with its generation) so + // a concurrent stopAllChainIndexers() can both wait for it to settle and abort its DB probe, + // rather than racing past it or hanging on it. + const probe = { generation, controller: new AbortController() } + this.startProbeControllers.add(probe) + const startPromise = this.doStartChainIndexer( + chainID, + generation, + probe.controller.signal + ) + this.pendingStarts.add(startPromise) + try { + return await startPromise + } finally { + this.pendingStarts.delete(startPromise) + this.startProbeControllers.delete(probe) + } + } + + private async doStartChainIndexer( + chainID: number, + generation: number, + probeSignal: AbortSignal + ): Promise { + // A standalone start (admin START_THREAD, resetCrawling) does not go through + // startAllChainIndexers()'s teardown wait, so wait here too: a previous OceanNode instance's + // teardown (its indexer stop + provider destroy) must finish before this indexer starts + // crawling, or two indexers run concurrently against the same DB/chain. awaitPendingTeardown() + // is a no-op when nothing is pending, so the batch path (which already awaited it) is unaffected. + await OceanNode.awaitPendingTeardown() + if (this.isStartCanceled(generation)) { + INDEXER_LOGGER.debug( + `Stop requested during startup (awaiting teardown); not starting indexer for chain ${chainID}` + ) + return null + } + // If an indexer is already running, stop it first const existingIndexer = this.indexers.get(chainID) if (existingIndexer && existingIndexer.isIndexing()) { @@ -288,7 +499,12 @@ export class OceanIndexer { INDEXER_LOGGER.error(`Unable to get Blockchain instance for chain: ${chainID}`) return null } - const canStartIndexer = await this.retryCrawlerWithDelay(blockchain) + const canStartIndexer = await this.retryCrawlerWithDelay( + blockchain, + 5000, + generation, + probeSignal + ) if (!canStartIndexer) { INDEXER_LOGGER.error( `Cannot start indexer for chain ${chainID}. Check DB and RPC connections!` @@ -296,6 +512,16 @@ export class OceanIndexer { return null } + // A stop may have been requested while retryCrawlerWithDelay was waiting for the connection. + // Do NOT start crawling in that case — otherwise this indexer outlives the stop and keeps + // polling the chain with a now-stale config (see `startGeneration`/`canceledGeneration`). + if (this.isStartCanceled(generation)) { + INDEXER_LOGGER.debug( + `Stop requested while connecting; not starting indexer for chain ${chainID}` + ) + return null + } + // Create new ChainIndexer instance const indexer = new ChainIndexer( blockchain, @@ -312,18 +538,47 @@ export class OceanIndexer { // Start indexing (runs in background, doesn't block) await indexer.start() + // ChainIndexer.start() launches the background crawl loop but does NOT register the indexer in + // `this.indexers`, so until the set() below a concurrent stopAllChainIndexers() cannot reach + // it. If a stop landed during start(), stop this freshly started indexer here instead of + // leaving it crawling past the stop (see `startGeneration`/`canceledGeneration`). + if (this.isStartCanceled(generation)) { + INDEXER_LOGGER.debug( + `Stop requested while starting; stopping freshly started indexer for chain ${chainID}` + ) + await indexer.stop() + return null + } + this.indexers.set(chainID, indexer) return indexer } // Start all chain indexers public async startAllChainIndexers(): Promise { + // Allocate this batch's generation BEFORE any await, so a stop that lands while we are still + // awaiting teardown/reindex is recorded against this generation and aborts the batch + // (see `startGeneration`/`canceledGeneration`). + const generation = ++this.startGeneration + // Close the overlap at its source: wait for a previous OceanNode instance's teardown // (its indexer stop + provider destroy) to finish before this indexer starts crawling, // so two indexers never run concurrently against the same DB/chain. await OceanNode.awaitPendingTeardown() + if (this.isStartCanceled(generation)) { + INDEXER_LOGGER.debug( + 'Stop requested during startup (awaiting teardown); aborting start of all chain indexers' + ) + return false + } await this.checkAndTriggerReindexing() + if (this.isStartCanceled(generation)) { + INDEXER_LOGGER.debug( + 'Stop requested during startup (reindex check); aborting start of all chain indexers' + ) + return false + } // Setup event listeners for all chains (they all use the same event emitter) this.setupEventListeners() @@ -331,8 +586,18 @@ export class OceanIndexer { // Start all indexers - they will run concurrently via async/await let count = 0 for (const network of this.supportedChains) { + // Stop the batch promptly if a stop landed mid-loop, instead of running a network/DB probe + // per remaining chain only for each start to self-abort on the canceled generation. + if (this.isStartCanceled(generation)) { + INDEXER_LOGGER.debug( + 'Stop requested during startup (starting chains); aborting remaining chain indexers' + ) + break + } const chainId = parseInt(network) - const indexer = await this.startChainIndexer(chainId) + // Share this batch's generation across all chains, so a single stop cancels the whole + // batch — including chains still coming up. + const indexer = await this.startChainIndexer(chainId, generation) if (indexer) { count++ } @@ -341,7 +606,27 @@ export class OceanIndexer { return count === this.supportedChains.length } + // Register a crawling-event handler and track it so stop() can detach exactly this instance's set. + private onCrawling(event: string, handler: (payload: any) => void): void { + this.crawlingEventHandlers.push({ event, handler }) + INDEXER_CRAWLING_EVENT_EMITTER.on(event, handler) + } + + // Detach every crawling-event handler this instance registered (NOT removeAllListeners(), which + // would also drop handlers belonging to other OceanIndexers sharing the module-level emitter). + private teardownEventListeners(): void { + for (const { event, handler } of this.crawlingEventHandlers) { + INDEXER_CRAWLING_EVENT_EMITTER.removeListener(event, handler) + } + this.crawlingEventHandlers = [] + } + private setupEventListeners() { + // Detach any previously registered handlers first: setupEventListeners() runs on every + // startAllChainIndexers() (including DB-reconnect restarts), so without this each restart would + // stack another full set of handlers on the shared emitter (duplicate DDO emissions, leaks). + this.teardownEventListeners() + // Listen to metadata events from any chain indexer const metadataEvents = [ EVENTS.METADATA_CREATED, @@ -357,7 +642,7 @@ export class OceanIndexer { ] metadataEvents.forEach((eventType) => { - INDEXER_CRAWLING_EVENT_EMITTER.on(eventType, async (event: any) => { + this.onCrawling(eventType, async (event: any) => { try { if (!event.data) { INDEXER_LOGGER.debug( @@ -379,49 +664,40 @@ export class OceanIndexer { }) // Listen to reindex queue pop events - INDEXER_CRAWLING_EVENT_EMITTER.on( - INDEXER_CRAWLING_EVENTS.REINDEX_QUEUE_POP, - (event: any) => { - try { - INDEXING_QUEUE = INDEXING_QUEUE.filter( - (task) => task.txId !== event.txId && task.chainId !== event.chainId - ) - this.updateJobStatus( - PROTOCOL_COMMANDS.REINDEX_TX, - create256Hash([event.chainId, event.txId].join('')), - CommandStatus.SUCCESS - ) - } catch (err) { - INDEXER_LOGGER.error(`Reindex queue pop handler failed: ${err?.message ?? err}`) - } + this.onCrawling(INDEXER_CRAWLING_EVENTS.REINDEX_QUEUE_POP, (event: any) => { + try { + INDEXING_QUEUE = INDEXING_QUEUE.filter( + (task) => task.txId !== event.txId && task.chainId !== event.chainId + ) + this.updateJobStatus( + PROTOCOL_COMMANDS.REINDEX_TX, + create256Hash([event.chainId, event.txId].join('')), + CommandStatus.SUCCESS + ) + } catch (err) { + INDEXER_LOGGER.error(`Reindex queue pop handler failed: ${err?.message ?? err}`) } - ) + }) // Listen to reindex chain events - INDEXER_CRAWLING_EVENT_EMITTER.on( - INDEXER_CRAWLING_EVENTS.REINDEX_CHAIN, - (event: any) => { - try { - this.updateJobStatus( - PROTOCOL_COMMANDS.REINDEX_CHAIN, - create256Hash([event.chainId].join('')), - event.result ? CommandStatus.SUCCESS : CommandStatus.FAILURE - ) - } catch (err) { - INDEXER_LOGGER.error(`Reindex chain handler failed: ${err?.message ?? err}`) - } + this.onCrawling(INDEXER_CRAWLING_EVENTS.REINDEX_CHAIN, (event: any) => { + try { + this.updateJobStatus( + PROTOCOL_COMMANDS.REINDEX_CHAIN, + create256Hash([event.chainId].join('')), + event.result ? CommandStatus.SUCCESS : CommandStatus.FAILURE + ) + } catch (err) { + INDEXER_LOGGER.error(`Reindex chain handler failed: ${err?.message ?? err}`) } - ) + }) // Listen to crawling started events - INDEXER_CRAWLING_EVENT_EMITTER.on( - INDEXER_CRAWLING_EVENTS.CRAWLING_STARTED, - (event: any) => { - INDEXER_LOGGER.info( - `Chain ${event.chainId}: crawling from block ${event.startBlock}` - ) - } - ) + this.onCrawling(INDEXER_CRAWLING_EVENTS.CRAWLING_STARTED, (event: any) => { + INDEXER_LOGGER.info( + `Chain ${event.chainId}: crawling from block ${event.startBlock}` + ) + }) } public addReindexTask(reindexTask: ReindexTask): JobStatus | null { @@ -477,16 +753,18 @@ export class OceanIndexer { public getJobsPool(jobId?: string): JobStatus[] { if (jobId) { - const pos = -1 const result = this.filterJobs(jobId) - // if it finished, then we can remove it from the list of jobs + // if it finished, then we can remove it from the list of jobs. Use the job's actual index + // (result.position) returned by filterJobs — a hardcoded -1 spliced the LAST job instead, + // removing the wrong entry whenever the finished job was not at the end of the queue. if ( + result.position >= 0 && result.jobsResult.length === 1 && [CommandStatus.FAILURE, CommandStatus.SUCCESS].includes( result.jobsResult[0].status ) ) { - JOBS_QUEUE.splice(pos, 1) + JOBS_QUEUE.splice(result.position, 1) } return result.jobsResult } @@ -496,7 +774,10 @@ export class OceanIndexer { // when we add a new job, we change the status from DELIVERED to PENDING if still running after a couple secs public addJob(jobInfo: JobStatus) { JOBS_QUEUE.push(jobInfo) - setTimeout(() => { + // Track the timer so stop() can clear it; otherwise a torn-down OceanIndexer leaves a pending + // 2 s timer that fires after shutdown and mutates the shared JOBS_QUEUE. + const timer = setTimeout(() => { + this.jobTimers.delete(timer) const result = this.filterJobs(jobInfo.jobId) if ( result.jobsResult.length === 1 && @@ -505,6 +786,15 @@ export class OceanIndexer { this.updateJobStatus(jobInfo.command, jobInfo.hash, CommandStatus.PENDING) } }, 2000) + this.jobTimers.add(timer) + } + + // Cancel any pending addJob() status-promotion timers (called from stop()). + private clearJobTimers(): void { + for (const timer of this.jobTimers) { + clearTimeout(timer) + } + this.jobTimers.clear() } // filter jobs by job id, return the position of the job on the queue as well @@ -600,8 +890,18 @@ export class OceanIndexer { } public async stop(): Promise { + // Detach DB listeners and cancel any pending reconnect BEFORE stopping, so a + // CONNECTION_RESTORED arriving mid-stop cannot schedule a restart. stopAllChainIndexers() + // then marks in-flight starts (including an already-fired reconnect that is mid-startup) as + // canceled, so none can resurrect an indexer after shutdown. + this.teardownDbConnectionListeners() await this.stopAllChainIndexers() - INDEXER_CRAWLING_EVENT_EMITTER.removeAllListeners() - INDEXER_DDO_EVENT_EMITTER.removeAllListeners() + // Detach only THIS instance's crawling-event handlers and cancel its pending job timers. Do + // NOT removeAllListeners() on the shared emitters: INDEXER_CRAWLING_EVENT_EMITTER is shared by + // any future OceanIndexer, and INDEXER_DDO_EVENT_EMITTER has external subscribers (the P2P + // node's METADATA_CREATED handler, test utils) that a blanket removeAllListeners() would + // silently unsubscribe. + this.teardownEventListeners() + this.clearJobTimers() } } diff --git a/src/components/core/admin/IndexingThreadHandler.ts b/src/components/core/admin/IndexingThreadHandler.ts index 8c6eddb30..11d1b40ff 100644 --- a/src/components/core/admin/IndexingThreadHandler.ts +++ b/src/components/core/admin/IndexingThreadHandler.ts @@ -38,26 +38,40 @@ export class IndexingThreadHandler extends AdminCommandHandler { } CORE_LOGGER.info(`Admin ${task.action} for chain ${task.chainId ?? 'all'}`) if (task.action === IndexingCommand.START_THREAD) { - const output = task.chainId - ? indexer.startChainIndexer(task.chainId) - : indexer.startAllChainIndexers() + // Await the start: these are async operations (they can sleep for seconds while the chain + // connection comes up). Without the await `output` was always a truthy Promise, so the + // handler reported HTTP 200 before the indexer had actually started (or failed). A per-chain + // start resolves to the ChainIndexer or null; a start-all resolves to a boolean. + const success = task.chainId + ? (await indexer.startChainIndexer(task.chainId)) !== null + : (await indexer.startAllChainIndexers()) === true return { status: { - httpStatus: output ? 200 : 400, - error: output ? null : 'Unable to start indexing thread(s)!' + httpStatus: success ? 200 : 400, + error: success ? null : 'Unable to start indexing thread(s)!' }, - stream: output ? new ReadableString('OK') : null + stream: success ? new ReadableString('OK') : null } } else if (task.action === IndexingCommand.STOP_THREAD) { - const output = task.chainId - ? indexer.stopChainIndexer(task.chainId) - : indexer.stopAllChainIndexers() + // Await the stop for the same reason. stopChainIndexer resolves to void (success unless it + // throws); stopAllChainIndexers resolves to a boolean. + let success = true + try { + if (task.chainId) { + await indexer.stopChainIndexer(task.chainId) + } else { + success = (await indexer.stopAllChainIndexers()) === true + } + } catch (error) { + CORE_LOGGER.error(`Unable to stop indexing thread(s): ${error?.message ?? error}`) + success = false + } return { status: { - httpStatus: output ? 200 : 400, - error: output ? null : 'Unable to stop indexing thread(s)!' + httpStatus: success ? 200 : 400, + error: success ? null : 'Unable to stop indexing thread(s)!' }, - stream: output ? new ReadableString('OK') : null + stream: success ? new ReadableString('OK') : null } } } diff --git a/src/utils/database.ts b/src/utils/database.ts index cad5d5745..54f3cb269 100644 --- a/src/utils/database.ts +++ b/src/utils/database.ts @@ -23,10 +23,16 @@ export function hasValidDBConfiguration(configuration: OceanNodeDBConfig): boole return true } -// we can use this to check if DB connection is available -export async function isReachableConnection(url: string): Promise { +// we can use this to check if DB connection is available. An optional AbortSignal lets the caller +// cancel a probe that would otherwise hang on a URL that accepts the connection but never responds +// (the indexer aborts it when the owning start is stopped). With no signal the behaviour is +// unchanged: the probe resolves true/false purely on the fetch outcome, however long that takes. +export async function isReachableConnection( + url: string, + signal?: AbortSignal +): Promise { try { - await fetch(url) + await fetch(url, signal ? { signal } : undefined) return true } catch (error) { return false