From d0f44bf5203c36953bc4144122aac8d50f62217c Mon Sep 17 00:00:00 2001 From: Stephen Zhou <38493346+hyoban@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:05:52 +0800 Subject: [PATCH] fix(rolldown): bound memory during plugin detail hydration --- .../rolldown/__tests__/events-reader.test.ts | 102 +++++++++++ .../src/node/rolldown/events-reader.ts | 170 ++++++++++-------- 2 files changed, 199 insertions(+), 73 deletions(-) diff --git a/packages/rolldown/src/node/rolldown/__tests__/events-reader.test.ts b/packages/rolldown/src/node/rolldown/__tests__/events-reader.test.ts index c46ff1852..b2e1b8e87 100644 --- a/packages/rolldown/src/node/rolldown/__tests__/events-reader.test.ts +++ b/packages/rolldown/src/node/rolldown/__tests__/events-reader.test.ts @@ -38,3 +38,105 @@ describe('rolldown events reader', () => { } }) }) + +describe('plugin detail hydration', () => { + it.each([ + ['many small calls', 513, () => 10], + ['large calls', 17, () => 100_000], + ['one oversized call', 5, (i: number) => i === 2 ? 2 * 1024 * 1024 : 10], + ] as const)('bounds event reads for %s and preserves filtered, sorted metrics', async (_, count, size) => { + const reader = RolldownEventsReader.get('/mock/plugin-batches/logs.json') + const entries = Array.from({ length: count }, (_, i) => [String(i), { + start: { offset: i * 2, length: size(i) }, + end: { offset: i * 2 + 1, length: size(i) }, + }] as const) + const split = Math.ceil(count / 2) + Object.defineProperty(reader, 'moduleEventIndex', { value: new Map([ + ['a.ts', { resolveIds: new Map(), loads: new Map(entries.slice(0, split)), transforms: new Map() }], + ['b.ts', { resolveIds: new Map(), loads: new Map(entries.slice(split)), transforms: new Map() }], + ]) }) + Object.defineProperty(reader, 'metricsSummaryOnly', { value: true }) + vi.spyOn(reader, 'read').mockResolvedValue() + const read = vi.fn(async (locations: Array<{ offset: number, length: number }>) => { + expect(locations.length).toBeLessThanOrEqual(256) + if (locations.reduce((sum, location) => sum + location.length, 0) > 1024 * 1024) + expect(locations).toHaveLength(2) + return locations.map(({ offset }) => { + const i = Math.floor(offset / 2) + return offset % 2 + ? { action: 'HookLoadCallEnd', plugin_id: i % 2, plugin_name: `plugin-${i % 2}`, timestamp: 1000 - i, content: null } + : { action: 'HookLoadCallStart', timestamp: 999 - i } + }) + }) + Object.defineProperty(reader, 'readEventsAt', { value: read }) + try { + await reader.hydratePluginBuildMetrics(1) + const metrics = reader.manager.plugin_build_metrics.get(1)! + const expectedIds = entries.map(([id]) => id).filter(id => Number(id) % 2).reverse() + expect(metrics.calls.map(call => call.id)).toEqual(expectedIds) + expect(metrics.calls[0]).toMatchObject({ module: 'b.ts', duration: 1, unchanged: true, plugin_id: 1 }) + expect(metrics.calls.at(-1)).toMatchObject({ module: 'a.ts', duration: 1, unchanged: true }) + expect(read.mock.calls.length).toBeGreaterThan(1) + await reader.hydratePluginBuildMetrics(99) + expect(reader.manager.plugin_build_metrics.has(99)).toBe(false) + } + finally { + reader.dispose() + } + }) + + it('compares transform content in batches without changing equality results', async () => { + const reader = RolldownEventsReader.get('/mock/plugin-transforms/logs.json') + const calls = Array.from({ length: 300 }, (_, i) => ({ type: 'transform' as const, id: String(i), module: 'a.ts', plugin_id: 1, plugin_name: 'test', duration: 1, timestamp_start: i, timestamp_end: i + 1 })) + reader.manager.plugin_build_metrics.set(1, { plugin_id: 1, plugin_name: 'test', calls }) + vi.spyOn(reader, 'read').mockResolvedValue() + Object.defineProperty(reader, 'moduleEventIndex', { value: new Map([ + ['a.ts', { transforms: new Map(calls.map((call, i) => [call.id, { start: { offset: i * 2, length: 10 }, end: { offset: i * 2 + 1, length: 10 } }])) }], + ]) }) + const read = vi.fn(async (locations: Array<{ offset: number }>) => { + expect(locations.length).toBeLessThanOrEqual(256) + return locations.map(({ offset }) => ({ + action: offset % 2 ? 'HookTransformCallEnd' : 'HookTransformCallStart', + content: offset % 4 === 3 ? 'changed' : 'original', + })) + }) + Object.defineProperty(reader, 'readEventsAt', { value: read }) + try { + await reader.hydratePluginBuildMetrics(1) + expect(reader.manager.plugin_build_metrics.get(1)!.calls.map(call => call.unchanged)).toEqual(calls.map((_, i) => i % 2 === 0)) + expect(read).toHaveBeenCalledTimes(3) + await reader.hydratePluginBuildMetrics(1) + expect(read).toHaveBeenCalledTimes(3) + } + finally { + reader.dispose() + } + }) + + it('resolves string references per transform batch and handles missing index entries', async () => { + const reader = RolldownEventsReader.get('/mock/plugin-refs/logs.json') + const calls = Array.from({ length: 260 }, (_, i) => ({ type: 'transform' as const, id: String(i), module: 'a.ts', plugin_id: 1, plugin_name: 'test', duration: 1, timestamp_start: i, timestamp_end: i + 1 })) + reader.manager.plugin_build_metrics.set(1, { plugin_id: 1, plugin_name: 'test', calls }) + vi.spyOn(reader, 'read').mockResolvedValue() + Object.defineProperty(reader, 'moduleEventIndex', { value: new Map([ + ['a.ts', { transforms: new Map(calls.slice(0, -1).map((call, i) => [call.id, { start: { offset: i * 2, length: 10 }, end: { offset: i * 2 + 1, length: 10 } }])) }], + ]) }) + Object.defineProperty(reader, 'readEventsAt', { value: async (locations: Array<{ offset: number }>) => locations.map(({ offset }) => ({ + action: offset % 2 ? 'HookTransformCallEnd' : 'HookTransformCallStart', + content: `$ref:${offset}`, + })) }) + const readRefs = vi.fn(async (refs: Set) => { + expect(refs.size).toBeLessThanOrEqual(256) + return new Map([...refs].map(ref => [ref, Number(ref) % 4 === 3 ? 'changed' : 'original'])) + }) + Object.defineProperty(reader, 'readStringRefs', { value: readRefs }) + try { + await reader.hydratePluginBuildMetrics(1) + expect(reader.manager.plugin_build_metrics.get(1)!.calls.map(call => call.unchanged)).toEqual(calls.map((_, i) => i < 259 && i % 2 === 0)) + expect(readRefs).toHaveBeenCalledTimes(3) + } + finally { + reader.dispose() + } + }) +}) diff --git a/packages/rolldown/src/node/rolldown/events-reader.ts b/packages/rolldown/src/node/rolldown/events-reader.ts index 66a41ea4d..ff0d6ef1b 100644 --- a/packages/rolldown/src/node/rolldown/events-reader.ts +++ b/packages/rolldown/src/node/rolldown/events-reader.ts @@ -14,6 +14,8 @@ const readers: Map = new Map() const MAX_READERS = 32 const MAX_MODULE_METRICS_CACHE = 32 const MAX_MODULE_METRICS_CACHE_BYTES = 64 * 1024 * 1024 +const MAX_PLUGIN_METRICS_BATCH_CALLS = 128 +const MAX_PLUGIN_METRICS_BATCH_BYTES = 1024 * 1024 const READ_STREAM_HIGH_WATER_MARK = 1024 * 1024 const LINE_FEED = '\n'.charCodeAt(0) const CARRIAGE_RETURN = '\r'.charCodeAt(0) @@ -895,52 +897,56 @@ export class RolldownEventsReader { if (!transformCalls.length) return - const refs = new Set() - const transformEventIndexes: Array = [] - const locations: LineLocation[] = [] - - for (const call of transformCalls) { - const location = this.moduleEventIndex.get(call.module)?.transforms.get(call.id) - if (!location) { - transformEventIndexes.push(undefined) - continue + // Release event payloads and resolved strings between groups of transforms. + for (let offset = 0; offset < transformCalls.length; offset += MAX_PLUGIN_METRICS_BATCH_CALLS) { + const batch = transformCalls.slice(offset, offset + MAX_PLUGIN_METRICS_BATCH_CALLS) + const refs = new Set() + const transformEventIndexes: Array = [] + const locations: LineLocation[] = [] + + for (const call of batch) { + const location = this.moduleEventIndex.get(call.module)?.transforms.get(call.id) + if (!location) { + transformEventIndexes.push(undefined) + continue + } + transformEventIndexes.push(locations.length) + locations.push(location.start, location.end) } - transformEventIndexes.push(locations.length) - locations.push(location.start, location.end) - } - const events = await this.readEventsAt(locations) - const contents = transformCalls.map((call, index) => { - const eventIndex = transformEventIndexes[index] - if (eventIndex == null) { + const events = await this.readEventsAt(locations) + const contents = batch.map((call, index) => { + const eventIndex = transformEventIndexes[index] + if (eventIndex == null) { + return { + call, + content_from: undefined, + content_to: undefined, + } + } + const start = events[eventIndex] + const end = events[eventIndex + 1] return { call, - content_from: undefined, - content_to: undefined, + content_from: start?.action === 'HookTransformCallStart' + ? getDeferredContent(start, refs) + : { value: null, ref: null }, + content_to: end?.action === 'HookTransformCallEnd' + ? getDeferredContent(end, refs) + : { value: null, ref: null }, } - } - const start = events[eventIndex] - const end = events[eventIndex + 1] - return { - call, - content_from: start?.action === 'HookTransformCallStart' - ? getDeferredContent(start, refs) - : { value: null, ref: null }, - content_to: end?.action === 'HookTransformCallEnd' - ? getDeferredContent(end, refs) - : { value: null, ref: null }, - } - }) + }) - const refValues = await this.readStringRefs(refs) - for (const item of contents) { - if (!item.content_from || !item.content_to) { - item.call.unchanged = false - continue + const refValues = await this.readStringRefs(refs) + for (const item of contents) { + if (!item.content_from || !item.content_to) { + item.call.unchanged = false + continue + } + const contentFrom = resolveDeferredContent(item.content_from, refValues) + const contentTo = resolveDeferredContent(item.content_to, refValues) + item.call.unchanged = getContentHash(contentFrom) === getContentHash(contentTo) } - const contentFrom = resolveDeferredContent(item.content_from, refValues) - const contentTo = resolveDeferredContent(item.content_to, refValues) - item.call.unchanged = getContentHash(contentFrom) === getContentHash(contentTo) } } @@ -1097,48 +1103,66 @@ export class RolldownEventsReader { location: IndexedHookCall }> = [] - for (const [module, index] of this.moduleEventIndex) { - for (const [id, location] of index.resolveIds) - calls.push({ module, type: 'resolve', id, location }) - for (const [id, location] of index.loads) - calls.push({ module, type: 'load', id, location }) - for (const [id, location] of index.transforms) - calls.push({ module, type: 'transform', id, location }) - } - - const events = await this.readIndexedHookEvents(calls.map(call => [call.id, call.location])) const metrics: PluginBuildMetrics = { plugin_id: pluginId, plugin_name: '', calls: [], } - for (const [index, item] of events.entries()) { - const call = calls[index]! - const start = item.start - const end = item.end - if (!start || !end || !('plugin_id' in end) || end.plugin_id !== pluginId) - continue + // The index has no plugin IDs, so scan it in bounded batches and keep only + // the requested plugin's compact metrics. An oversized hook is read alone. + let bytes = 0 + const flush = async () => { + if (!calls.length) + return + const events = await this.readIndexedHookEvents(calls.map(call => [call.id, call.location])) + for (const [index, item] of events.entries()) { + const call = calls[index]! + const start = item.start + const end = item.end + if (!start || !end || !('plugin_id' in end) || end.plugin_id !== pluginId) + continue - const timestamp_start = 'timestamp' in start ? +start.timestamp : 0 - const timestamp_end = 'timestamp' in end ? +end.timestamp : 0 - metrics.plugin_name = end.plugin_name - metrics.calls.push({ - type: call.type, - id: call.id, - duration: timestamp_end - timestamp_start, - plugin_id: pluginId, - plugin_name: end.plugin_name, - module: call.type === 'resolve' && start.action === 'HookResolveIdCallStart' - ? start.module_request - : call.module, - timestamp_start, - timestamp_end, - unchanged: call.type === 'load' && end.action === 'HookLoadCallEnd' - ? !end.content - : undefined, - }) + const timestamp_start = 'timestamp' in start ? +start.timestamp : 0 + const timestamp_end = 'timestamp' in end ? +end.timestamp : 0 + metrics.plugin_name = end.plugin_name + metrics.calls.push({ + type: call.type, + id: call.id, + duration: timestamp_end - timestamp_start, + plugin_id: pluginId, + plugin_name: end.plugin_name, + module: call.type === 'resolve' && start.action === 'HookResolveIdCallStart' + ? start.module_request + : call.module, + timestamp_start, + timestamp_end, + unchanged: call.type === 'load' && end.action === 'HookLoadCallEnd' + ? !end.content + : undefined, + }) + } + + calls.length = 0 + bytes = 0 + } + + for (const [module, index] of this.moduleEventIndex) { + for (const [type, entries] of [ + ['resolve', index.resolveIds], + ['load', index.loads], + ['transform', index.transforms], + ] as const) { + for (const [id, location] of entries) { + const size = location.start.length + location.end.length + if (calls.length && (calls.length >= MAX_PLUGIN_METRICS_BATCH_CALLS || bytes + size > MAX_PLUGIN_METRICS_BATCH_BYTES)) + await flush() + calls.push({ module, type, id, location }) + bytes += size + } + } } + await flush() metrics.calls.sort((a, b) => a.timestamp_start - b.timestamp_start) return metrics.calls.length ? metrics : undefined