diff --git a/Dockerfile b/Dockerfile index a592c193..b026b62d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,4 +36,4 @@ RUN if echo "$BASE_IMAGE" | grep -q "node-caged"; then \ echo "export NODE_CAGED=false" > /app/.runtime-env; \ fi -CMD ["sh", "-c", ". /app/.runtime-env && exec node --expose-gc /app/server/index.mjs"] \ No newline at end of file +CMD ["sh", "-c", ". /app/.runtime-env && exec node /app/server/index.mjs"] \ No newline at end of file diff --git a/README.md b/README.md index 90307017..32120c11 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,10 @@ volumes: cache-data: ``` +Multipart downloads and background merges use stream backpressure and normal V8 garbage collection. +The server does not require Node's `--expose-gc` flag or force collection between parts. +Aggregate memory usage still depends on workload concurrency and configured storage buffers. + ## Documentation 👉 👈 diff --git a/lib/storage.ts b/lib/storage.ts index 02a4519d..9296766c 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -150,8 +150,6 @@ export class Storage { responseStream.end() mergerStream.end() - - await globalThis.gc?.() } private async *streamParts(location: StorageLocation, signal?: AbortSignal) { @@ -170,8 +168,6 @@ export class Storage { } finally { partStream.destroy() } - - await globalThis.gc?.() } } diff --git a/plugins/setup.ts b/plugins/setup.ts index 783d73e2..b9975b34 100644 --- a/plugins/setup.ts +++ b/plugins/setup.ts @@ -12,11 +12,6 @@ export default defineNitroPlugin(async (nitro) => { if (cluster.isPrimary) { logger.info(`🚀 Starting GitHub Actions Cache Server (${version})`) - if (!globalThis.gc) - logger.warn( - 'Garbage collection is not exposed. Start the process with `node --expose-gc` for improved memory usage under high load.', - ) - for (const signal of ['SIGTERM', 'SIGINT'] satisfies NodeJS.Signals[]) { process.on(signal, () => { const workers = Object.values(cluster.workers ?? {}) diff --git a/tests/cleanup-lifecycle.test.ts b/tests/cleanup-lifecycle.test.ts index 73dba28c..82b26261 100644 --- a/tests/cleanup-lifecycle.test.ts +++ b/tests/cleanup-lifecycle.test.ts @@ -12,6 +12,89 @@ describe('cleanup lifecycle', () => { }) afterAll(() => vi.unstubAllGlobals()) + test('concurrent multipart restores preserve exact bytes before and after merging', async () => { + const db = await getDatabase() + const storage = await Storage.fromEnv() + const entries = Array.from({ length: 4 }, (_, entryIndex) => ({ + key: randomUUID(), + version: 'v1', + scope: 'refs/heads/main', + repoId: '123', + parts: Array.from({ length: 8 }, (_, partIndex) => + Buffer.from( + Array.from( + { length: 16 * 1024 + partIndex }, + (_, offset) => (entryIndex * 31 + partIndex * 17 + offset) % 256, + ), + ), + ), + })) + + try { + const caches = [] + for (const entry of entries) { + const upload = await storage.createUpload(entry) + expect(upload).toBeDefined() + for (const [partIndex, part] of entry.parts.entries()) + await storage.uploadPart(upload!.id, partIndex, Readable.from(part)) + await storage.completeUpload(entry) + const cache = await storage.matchCacheEntry({ + keys: [entry.key], + version: entry.version, + scopes: [entry.scope], + repoId: entry.repoId, + }) + expect(cache).toBeDefined() + caches.push({ cache: cache!.match, expected: Buffer.concat(entry.parts) }) + } + + for (const phase of ['cold', 'warm']) { + await Promise.all( + caches.map(async ({ cache, expected }) => { + const stream = await storage.download(cache.id) + expect(stream, phase).toBeDefined() + const chunks: Buffer[] = [] + for await (const chunk of stream!) chunks.push(Buffer.from(chunk)) + expect(Buffer.concat(chunks), phase).toEqual(expected) + }), + ) + await storage.waitForOngoingMerges() + for (const { cache } of caches) { + const location = await db + .selectFrom('storage_locations') + .where('id', '=', cache.locationId) + .select('mergedAt') + .executeTakeFirstOrThrow() + expect(location.mergedAt, phase).not.toBeNull() + } + } + } finally { + await storage.waitForOngoingMerges() + const locations = await db + .selectFrom('cache_entries') + .innerJoin('storage_locations', 'storage_locations.id', 'cache_entries.locationId') + .where( + 'cache_entries.key', + 'in', + entries.map(({ key }) => key), + ) + .select(['storage_locations.id', 'storage_locations.folderName']) + .execute() + for (const { id, folderName } of locations) { + await db.deleteFrom('storage_locations').where('id', '=', id).execute() + await storage.adapter.deleteFolder(folderName) + } + await db + .deleteFrom('uploads') + .where( + 'key', + 'in', + entries.map(({ key }) => key), + ) + .execute() + } + }) + test('evicts least-recently-used entries to 90% only after exceeding the budget', async () => { const db = await getDatabase() const storage = await Storage.fromEnv()