From 789eb4f24f30827005a3079710ef389e81f707f9 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 17 Nov 2026 16:35:51 +0200 Subject: [PATCH 1/3] Service logs were only served while the container existed --- docs/API.md | 21 +- docs/services.md | 36 +- src/@types/C2D/ServiceOnDemand.ts | 16 + src/components/c2d/compute_engine_base.ts | 6 +- src/components/c2d/compute_engine_docker.ts | 160 +++++- src/components/c2d/serviceLogs.ts | 184 +++++++ .../core/service/getStreamableLogs.ts | 5 +- src/components/core/service/utils.ts | 4 +- src/components/database/sqliteCompute.ts | 11 +- src/test/integration/services.test.ts | 27 + src/test/unit/service/serviceHandlers.test.ts | 3 +- .../unit/service/serviceJobsDatabase.test.ts | 28 ++ .../unit/service/serviceLogsArchive.test.ts | 476 ++++++++++++++++++ .../service/serviceOutputsArchive.test.ts | 4 +- src/test/unit/service/serviceUtils.test.ts | 20 + 15 files changed, 952 insertions(+), 49 deletions(-) create mode 100644 src/components/c2d/serviceLogs.ts create mode 100644 src/test/unit/service/serviceLogsArchive.test.ts diff --git a/docs/API.md b/docs/API.md index 08f9249f1..d15d1a0c9 100644 --- a/docs/API.md +++ b/docs/API.md @@ -2965,10 +2965,13 @@ until `expiresAt`. Only `release: true` ends the paid window early. #### Description -Stream the service container's stdout/stderr logs live. **Authenticated and owner-scoped** +Stream the service container's stdout/stderr logs. **Authenticated and owner-scoped** — only the service's owner (`consumerAddress`, proven by signature/nonce or auth token) can -read its logs. Available while the service is `Running` (`40`) or `Error` (`99`) — a crashed -container is kept around until `stop`/`restart`, so its logs remain fetchable for diagnosis. +read its logs. While the service has a container — `Running` (`40`), or `Error` (`99`): a +crashed container is kept around until `stop`/`restart` — its log is followed live. Once the +container is gone, the logs the node saved when each of the service's containers was removed +are served instead, oldest first, until `storageExpiry` seconds after `expiresAt`. Only the last +16 MiB of each container's log are kept. #### Query Parameters @@ -2982,10 +2985,11 @@ container is kept around until `stop`/`restart`, so its logs remain fetchable fo #### Response (200) -Raw `stdout`/`stderr` byte stream from the container, connection kept open and followed live. -With `since` set, historical output before that point is skipped — useful for a long-lived -service where fetching the full history would otherwise dump days/weeks of buffered logs -before reaching the live tail (e.g. `since=1h` for just the last hour). +Raw `stdout`/`stderr` byte stream from the container (Docker-multiplexed frames), connection +kept open and followed live. Saved logs come in the same format, and the stream ends after the +last line. With `since` set, historical output before that point is skipped — useful for a +long-lived service where fetching the full history would otherwise dump days/weeks of buffered +logs before reaching the live tail (e.g. `since=1h` for just the last hour). #### Response (400) @@ -2993,7 +2997,8 @@ before reaching the live tail (e.g. `since=1h` for just the last hour). #### Response (404) -Service not found, or not `Running`/`Error`. +Service not found, or it has no container and no saved logs (never saved, or deleted after +storage expiry). #### Response (401) diff --git a/docs/services.md b/docs/services.md index 12faf1ae4..86a8f13de 100644 --- a/docs/services.md +++ b/docs/services.md @@ -40,7 +40,7 @@ and `signature` as query parameters (or an auth-token `Authorization` header). | `SERVICE_RESTART` | `/api/services/serviceRestart` | POST | Recreate the container (no extra charge); asynchronous like start — returns once the job is `Restarting`, poll `serviceStatus`. Optionally restart on a **new image spec** (bug-fix flow) — see below | | `SERVICE_STOP` | `/api/services/serviceStop` | POST | Tear down the container; the paid resource reservation (cpu/ram/gpu + host ports) is kept until `expiresAt`, so the service can be restarted anytime on the same endpoints. `release: true` ends the paid window now and frees it instead (no refund, no restart) | | `SERVICE_GET_TEMPLATES` | `/api/services/serviceTemplates` | GET | List operator-published service templates | -| `SERVICE_GET_STREAMABLE_LOGS` | `/api/services/serviceStreamableLogs` | GET | Stream the container's live stdout/stderr logs — authenticated, owner-scoped; available while `Running` or `Error`; optional `since` to skip history | +| `SERVICE_GET_STREAMABLE_LOGS` | `/api/services/serviceStreamableLogs` | GET | Stream the container's stdout/stderr logs — authenticated, owner-scoped; followed live while the service has a container (`Running`, or `Error` before teardown), otherwise the logs saved when its containers were removed, until storage expiry; optional `since` to skip history. See [Service logs](#service-logs) | | `SERVICE_GET_RESULT` | `/api/services/serviceResult` | GET | Download the service's `/data/outputs` as a zip — one of its archives (`index`, resumable with `offset`) or the running container's (`live=true`); authenticated, owner-scoped. See [Service outputs](#service-outputs) | **Start is asynchronous.** `serviceStart` does only the fast, synchronous validation and then @@ -269,6 +269,31 @@ compute job's outputs: `outputArchives: []`. - There is no size cap on archives beyond the node's free disk. +## Service logs + +`serviceStreamableLogs` follows the container's stdout/stderr live while the service has a +container. Docker deletes a container's logs together with the container, so the node saves them +first, which keeps them available after the session ends: + +- **They are saved whenever a container of the service goes away**: `serviceStop`, the expiry + sweep, a `serviceRestart` (the old container), and the crash recovery at node start. This + happens for every service, including one with an output bucket, because the bucket only holds + `/data/outputs`. Each container gives one file, `logs-.log`, next to the output archives + under `/services//`. The files are listed on `serviceStatus` as + `logArchives` (`index`, `filename`, `filesize`, `createdAt`, `containerId`, and + `droppedBytes` when output was cut). Saving is best-effort: if it fails, teardown continues and + the node only logs the error. A container that logged nothing produces no file. +- **Only the last 16 MiB of each container's log are kept.** When a container logged more, its + oldest output is dropped, and the saved log starts with an `[ocean-node]` line on stderr that + says how much was dropped. +- **Once the service has no container, `serviceStreamableLogs` serves the saved logs**, oldest + container first, in the same multiplexed format as the live stream. The stream ends after the + last line instead of following. `since` works the same way. The route returns `404` only when + there is neither a container nor a saved log. While a container exists, only its own live log + is returned, so after a `serviceRestart` the previous containers' logs are not included. +- **Saved logs expire with the output archives**: `storageExpiry` seconds after `expiresAt`, by + the same cron. The service record is kept, with `logArchives: []`. + ## Configuration Service-on-demand is configured per Docker connection under `serviceOnDemand`: @@ -630,16 +655,17 @@ then `docker network rm t`. - **`serviceStreamableLogs` is authenticated and owner-scoped, like `serviceStatus`.** Container stdout/stderr can leak secrets or sensitive request data, so the same proof-of-`consumerAddress` + ownership check gates log access — a non-owner gets `401`. - Logs are only served while the service is `Running` or `Error` (a crashed container's - logs stay available for diagnosis until `stop`/`restart` tears it down); otherwise the - route returns `404`. By default the full history since container start is returned before + Logs are followed live while the service has a container (`Running`, or `Error` until + `stop`/`restart` tears the crashed container down). After that the logs saved at teardown are + served until storage expiry (see [Service logs](#service-logs)), and the route returns `404` + only when there are none. By default the full history since container start is returned before the stream switches to following live output — for a service that has been running for days or weeks that can be a lot of data, so pass `since` (a Unix timestamp, or a relative duration like `1h`) to skip straight to recent output. - **`serviceResult` is authenticated and owner-scoped, like `serviceStreamableLogs`.** A service's outputs are its owner's data; a non-owner gets the same `400` as an unknown - `serviceId`. The node-wide `serviceList` leaves out `outputArchives`. + `serviceId`. The node-wide `serviceList` leaves out `outputArchives` and `logArchives`. - **`allowImageBuild` runs arbitrary build instructions.** When enabled, a consumer's inline `dockerfile` is built by the Docker daemon, so its `RUN` steps execute arbitrary diff --git a/src/@types/C2D/ServiceOnDemand.ts b/src/@types/C2D/ServiceOnDemand.ts index 1737be8a0..2f9733683 100644 --- a/src/@types/C2D/ServiceOnDemand.ts +++ b/src/@types/C2D/ServiceOnDemand.ts @@ -201,6 +201,19 @@ export interface ServiceOutputArchive { containerId: string // container it was taken from — a retried teardown must not archive it twice } +// The stdout/stderr of one container of the service, saved right before it was removed (Docker +// drops a container's logs with it). SERVICE_GET_STREAMABLE_LOGS serves them, oldest first, once +// the service has no container, until expiresAt + the environment's storageExpiry. +export interface ServiceLogArchive { + index: number + filename: string // e.g. logs-0.log, under /services// + filesize: number // bytes + createdAt: number // Unix ms timestamp + containerId: string // container it was taken from — a retried teardown must not save it twice + // Bytes of the oldest output left out because the container logged more than the node keeps. + droppedBytes?: number +} + export interface ServiceEndpoint { containerPort: number hostPort: number @@ -304,6 +317,9 @@ export interface ServiceJob { outputBucketId?: string // persistent-storage bucket bind-mounted at /data/outputs // Archives of /data/outputs, only for services without an outputBucketId. outputArchives?: ServiceOutputArchive[] + // Saved logs of the containers that are gone, for every service (an output bucket holds only + // /data/outputs). + logArchives?: ServiceLogArchive[] // Internal, only while a restart is in flight: the stopped old container, kept until its // /data/outputs has been carried over into the new one (orphan recovery archives and // removes it if the node dies before that). Never returned to clients. diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index 9dff063a8..5e7ab6231 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -209,6 +209,8 @@ export abstract class C2DEngine { return [] } + // The live log of the service's container while it has one, else the logs saved when its + // containers were removed (see ServiceJob.logArchives). Null when there is neither. // eslint-disable-next-line require-await public async getServiceStreamableLogs( serviceId: string, @@ -231,8 +233,8 @@ export abstract class C2DEngine { return null } - // Deletes the output archives of expired services whose environment's storageExpiry has - // elapsed since expiresAt. Returns how many services had their archives deleted. + // Deletes the output archives and saved logs of expired services whose environment's + // storageExpiry has elapsed since expiresAt. Returns how many services had them deleted. // eslint-disable-next-line require-await public async cleanupExpiredServiceOutputs(): Promise { return 0 diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index bddae5de3..75a127ba7 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -125,6 +125,7 @@ import { tarToZip, type ServiceResult } from './serviceOutputsZip.js' +import { collectServiceLogs, savedServiceLogsStream } from './serviceLogs.js' import { buildServiceHfPlan, getServiceHfMirrorSettings, @@ -4615,6 +4616,84 @@ export class C2DEngineDocker extends C2DEngine { } } + // Saves the container's stdout/stderr to /services//logs-.log and records + // it on job.logArchives, so SERVICE_GET_STREAMABLE_LOGS can serve it once the container is gone: + // Docker drops a container's logs with the container. Must run after the container was stopped + // (its log is then final, and reading it ends by itself) and before it is removed. Unlike the + // outputs it runs for services with an output bucket too. Keeps the newest + // SERVICE_LOGS_MAX_BYTES. Best-effort — never throws, a failure is only logged; the caller + // persists the job. + private async saveServiceLogs( + job: ServiceJob, + container: Dockerode.Container + ): Promise { + const saved = job.logArchives ?? [] + // A teardown retried after a failure must not save the same container twice. + if (saved.some((a) => a.containerId === container.id)) return + const index = saved.reduce((next, a) => Math.max(next, a.index + 1), 0) + const filename = `logs-${index}.log` + let partialPath: string | undefined + try { + let source: NodeJS.ReadableStream + try { + // Timestamped, so a `since` filter still works on the saved file. + source = (await container.logs({ + stdout: true, + stderr: true, + follow: true, + timestamps: true + })) as unknown as NodeJS.ReadableStream + } catch (e: any) { + if (e?.statusCode === 404) { + CORE_LOGGER.debug(`service ${job.serviceId}: no logs to save: ${e.message}`) + return + } + throw e + } + const { data, droppedBytes, cutShort } = await collectServiceLogs(source) + if (cutShort) { + CORE_LOGGER.warn( + `service ${job.serviceId}: reading the logs of container ${container.id} timed out — saving what was read` + ) + } + if (!data.length) { + CORE_LOGGER.debug( + `service ${job.serviceId}: container ${container.id} logged nothing` + ) + return + } + const folder = this.getServiceOutputsFolder(job.serviceId) + const finalPath = path.join(folder, filename) + partialPath = finalPath + '.partial' + mkdirSync(folder, { recursive: true }) + writeFileSync(partialPath, data) + renameSync(partialPath, finalPath) + job.logArchives = [ + ...saved, + { + index, + filename, + filesize: data.length, + createdAt: Date.now(), + containerId: container.id, + ...(droppedBytes ? { droppedBytes } : {}) + } + ] + CORE_LOGGER.info( + `service ${job.serviceId}: saved the logs of container ${container.id} to ${filename}, ` + + `${data.length} bytes` + + (droppedBytes ? `, ${droppedBytes} older bytes dropped` : '') + ) + } catch (e: any) { + if (partialPath) rmSync(partialPath, { force: true }) + CORE_LOGGER.error( + `service ${job.serviceId}: failed to save the logs of container ${container.id}: ${ + e?.message ?? e + }` + ) + } + } + // Streams a container's /data/outputs into another one — created, not yet started — as a // tar straight between the two, so a restart keeps what the service wrote. The source may be // stopped and off its (already removed) network: Docker reads a stopped container's @@ -4804,12 +4883,14 @@ export class C2DEngineDocker extends C2DEngine { } const { previousContainerId } = job if (job.containerId || previousContainerId) { - // Only a claimed service ever ran — archive what it wrote before its containers go. - // A restart that died before switching to the new container (containerId still - // empty) left the results in the previous one it was carrying them over from. + // Only a claimed service ever ran — archive what it wrote, and save what it logged, + // before its containers go. A restart that died before switching to the new + // container (containerId still empty) left the results in the previous one it was + // carrying them over from. if (job.payment.claimTx) { const holder = this.docker.getContainer(job.containerId || previousContainerId) await holder.stop({ t: 10 }).catch(() => {}) + await this.saveServiceLogs(job, holder) await this.archiveServiceOutputs(job, holder) } if (job.containerId) @@ -6290,6 +6371,7 @@ export class C2DEngineDocker extends C2DEngine { await c.stop({ t: 10 }).catch((e) => { if (!isBenignDockerError(e)) throw e }) + await this.saveServiceLogs(job, c) await this.archiveServiceOutputs(job, c) // v: also drop the anonymous volumes the image declared (e.g. comfyui-boot's // VOLUME /root, where it keeps its models): no later container can attach them, @@ -6505,6 +6587,8 @@ export class C2DEngineDocker extends C2DEngine { if (!isBenignDockerError(e)) stopped = false logDockerCleanupError(e, `restart ${serviceId}: old container stop: ${e.message}`) }) + // Its logs end here, whichever way it goes below: the new container starts a fresh log. + await this.saveServiceLogs(job, c) if (job.outputBucketId) { await c.remove({ force: true, v: true }).catch((e) => { logDockerCleanupError( @@ -6799,27 +6883,49 @@ export class C2DEngineDocker extends C2DEngine { ): Promise { const [job] = await this.db.getServiceJob(serviceId, owner) if (!job) return null + // While the service has a container (Running, or Error before teardown) its live log is the + // answer: it covers everything that container logged. if ( - job.status !== ServiceStatusNumber.Running && - job.status !== ServiceStatusNumber.Error + job.containerId && + (job.status === ServiceStatusNumber.Running || + job.status === ServiceStatusNumber.Error) ) { - return null - } - try { - const container = this.docker.getContainer(job.containerId) - await container.inspect() // throws if the container no longer exists - return (await container.logs({ - stdout: true, - stderr: true, - follow: true, - ...(since !== undefined ? { since } : {}) - })) as unknown as NodeJS.ReadableStream - } catch (e: any) { - CORE_LOGGER.error( - `getServiceStreamableLogs failed for ${serviceId}: ${e?.message ?? e}` - ) - return null + try { + const container = this.docker.getContainer(job.containerId) + await container.inspect() // throws if the container no longer exists + return (await container.logs({ + stdout: true, + stderr: true, + follow: true, + ...(since !== undefined ? { since } : {}) + })) as unknown as NodeJS.ReadableStream + } catch (e: any) { + // Gone: fall back to the saved logs. Anything else is a Docker failure, and serving + // older saved logs in place of the live ones would hide it. + if (e?.statusCode !== 404) { + CORE_LOGGER.error( + `getServiceStreamableLogs failed for ${serviceId}: ${e?.message ?? e}` + ) + return null + } + } } + return this.openSavedServiceLogs(job, since) + } + + // The logs saved at teardown (job.logArchives), oldest container first, or null when there + // are none — never saved, or deleted with the output archives once storageExpiry elapsed. + private openSavedServiceLogs( + job: ServiceJob, + since?: number + ): NodeJS.ReadableStream | null { + const folder = this.getServiceOutputsFolder(job.serviceId) + const files = [...(job.logArchives ?? [])] + .sort((a, b) => a.index - b.index) + .map((a) => path.join(folder, a.filename)) + .filter((file) => existsSync(file)) + if (!files.length) return null + return savedServiceLogsStream(files, since) } public override async getServiceResult( @@ -6913,29 +7019,33 @@ export class C2DEngineDocker extends C2DEngine { now - shortest * 1000, this.getC2DConfig().hash ) + // The output archives and the saved logs share the service's folder and expire together. + const holdsFiles = (job?: ServiceJob) => + !!(job?.outputArchives?.length || job?.logArchives?.length) let cleaned = 0 for (const candidate of candidates) { - if (!candidate.outputArchives?.length) continue + if (!holdsFiles(candidate)) continue const expiry = storageExpiry.get(candidate.environment) ?? defaultStorageExpiry if (now < candidate.expiresAt + expiry * 1000) continue try { await this.runExclusive(candidate.serviceId, async () => { const [job] = await this.db.getServiceJob(candidate.serviceId, candidate.owner) - if (!job?.outputArchives?.length) return + if (!holdsFiles(job)) return rmSync(this.getServiceOutputsFolder(job.serviceId), { recursive: true, force: true }) job.outputArchives = [] + job.logArchives = [] await this.db.updateServiceJob(job) cleaned++ CORE_LOGGER.info( - `service ${job.serviceId}: output archives deleted (storage expiry elapsed)` + `service ${job.serviceId}: output archives and saved logs deleted (storage expiry elapsed)` ) }) } catch (e: any) { CORE_LOGGER.error( - `service ${candidate.serviceId}: failed to delete output archives: ${e.message} — retrying next run` + `service ${candidate.serviceId}: failed to delete output archives and saved logs: ${e.message} — retrying next run` ) } } diff --git a/src/components/c2d/serviceLogs.ts b/src/components/c2d/serviceLogs.ts new file mode 100644 index 000000000..416a9e404 --- /dev/null +++ b/src/components/c2d/serviceLogs.ts @@ -0,0 +1,184 @@ +import { createReadStream } from 'fs' +import { Readable } from 'stream' + +// The most of one container's log the node saves at teardown. A service can run for weeks, and +// Docker keeps everything it logged, so only the most recent output is kept: that is where a +// crash or a stop shows up. +export const SERVICE_LOGS_MAX_BYTES = 16 * 1024 * 1024 + +// Reading a stopped container's log ends by itself once Docker has sent it all. These bound the +// read when the container could not be stopped and is still logging. +export const SERVICE_LOGS_IDLE_MS = 5_000 +export const SERVICE_LOGS_MAX_MS = 60_000 + +// A non-TTY container's stdout and stderr come multiplexed: each write is a frame with an 8-byte +// header — [stream type, 0, 0, 0, payload size as uint32 BE] — followed by the payload. Service +// containers never get a TTY, so their logs always have this shape. +const FRAME_HEADER_BYTES = 8 +const STDERR = 2 + +export interface DockerLogFrame { + stream: number // 0 stdin, 1 stdout, 2 stderr (3 is Docker's own error stream) + payload: Buffer +} + +export function encodeDockerLogFrame(stream: number, payload: Buffer): Buffer { + const header = Buffer.alloc(FRAME_HEADER_BYTES) + header[0] = stream + header.writeUInt32BE(payload.length, 4) + return Buffer.concat([header, payload]) +} + +/** + * Yields the frames of a multiplexed Docker log stream. Throws on bytes that are not a frame + * header (e.g. the raw output of a TTY container). A frame cut off by the end of the stream is + * dropped. + */ +export async function* readDockerLogFrames( + source: AsyncIterable +): AsyncGenerator { + let pending: Buffer = Buffer.alloc(0) + for await (const chunk of source) { + const bytes = typeof chunk === 'string' ? Buffer.from(chunk) : chunk + pending = pending.length ? Buffer.concat([pending, bytes]) : bytes + let offset = 0 + while (pending.length - offset >= FRAME_HEADER_BYTES) { + const stream = pending[offset] + if ( + stream > 3 || + pending[offset + 1] || + pending[offset + 2] || + pending[offset + 3] + ) { + throw new Error('not a multiplexed Docker log stream') + } + const size = pending.readUInt32BE(offset + 4) + const end = offset + FRAME_HEADER_BYTES + size + if (end > pending.length) break + yield { stream, payload: pending.subarray(offset + FRAME_HEADER_BYTES, end) } + offset = end + } + pending = pending.subarray(offset) + } +} + +/** + * Splits off the RFC 3339 timestamp Docker puts in front of each entry when asked for + * `timestamps: true`. `ms` is undefined for a payload without one. + */ +export function splitLogTimestamp(payload: Buffer): { ms?: number; body: Buffer } { + const space = payload.indexOf(0x20) + // "2026-11-17T10:00:00.123456789Z" is 30 characters, a numeric offset makes it 35. + if (space >= 20 && space <= 40) { + const ms = Date.parse(payload.toString('latin1', 0, space)) + if (!Number.isNaN(ms)) return { ms, body: payload.subarray(space + 1) } + } + return { body: payload } +} + +export interface CollectedServiceLogs { + // The frames to save, still multiplexed and timestamped, or empty if the container logged + // nothing. Starts with a note on stderr when older output was dropped. + data: Buffer + droppedBytes: number + // The read was cut short by `idleMs`/`maxMs`, so the newest output may be missing. + cutShort: boolean +} + +/** + * Reads a container's whole log (`container.logs({ follow: true, timestamps: true })`) and keeps + * the newest `maxBytes` of it, dropping whole frames from the start. Holds at most `maxBytes` in + * memory. Gives up after `idleMs` without data or `maxMs` in all, keeping what it read. + */ +export async function collectServiceLogs( + source: NodeJS.ReadableStream, + { + maxBytes = SERVICE_LOGS_MAX_BYTES, + idleMs = SERVICE_LOGS_IDLE_MS, + maxMs = SERVICE_LOGS_MAX_MS + }: { maxBytes?: number; idleMs?: number; maxMs?: number } = {} +): Promise { + const frames: Array = [] + let head = 0 + let keptBytes = 0 + let droppedBytes = 0 + let cutShort = false + const giveUp = () => { + cutShort = true + ;(source as Readable).destroy?.() + } + const maxTimer = setTimeout(giveUp, maxMs) + const idleTimer = setTimeout(giveUp, idleMs) + try { + for await (const { stream, payload } of readDockerLogFrames( + source as AsyncIterable + )) { + idleTimer.refresh() + // A copy, so the kept frames don't pin the (larger) chunks they were read from. + frames.push({ stream, payload: Buffer.from(payload) }) + keptBytes += FRAME_HEADER_BYTES + payload.length + // Always keep the newest frame, even one bigger than the limit on its own. + while (keptBytes > maxBytes && head < frames.length - 1) { + const size = FRAME_HEADER_BYTES + frames[head].payload.length + frames[head] = undefined + head++ + keptBytes -= size + droppedBytes += size + } + if (head > 4096 && head * 2 > frames.length) { + frames.splice(0, head) + head = 0 + } + } + } catch (e) { + // Destroying the source to give up ends the read with an error; anything else is real. + if (!cutShort) throw e + } finally { + clearTimeout(maxTimer) + clearTimeout(idleTimer) + } + + const kept = frames.slice(head) as DockerLogFrame[] + const parts = kept.map((f) => encodeDockerLogFrame(f.stream, f.payload)) + if (droppedBytes > 0) { + // Stamped like the first kept entry, so a `since` filter keeps or drops it with that entry. + const { ms } = splitLogTimestamp(kept[0].payload) + const stamp = ms === undefined ? '' : `${new Date(ms).toISOString()} ` + const note = + `${stamp}[ocean-node] ${droppedBytes} bytes of older output were dropped: ` + + `the node keeps the last ${maxBytes} bytes of a container's log.\n` + parts.unshift(encodeDockerLogFrame(STDERR, Buffer.from(note))) + } + return { data: Buffer.concat(parts), droppedBytes, cutShort } +} + +/** + * Streams saved log files, oldest first, in the same multiplexed format as a live + * `container.logs()` stream. With `since` (Unix seconds, like Docker's own filter) older entries + * are left out. The timestamps the files were saved with are stripped, as a live stream has none. + */ +export function savedServiceLogsStream(files: string[], since?: number): Readable { + const sinceMs = since === undefined ? undefined : since * 1000 + async function* chunks(): AsyncGenerator { + let batch: Buffer[] = [] + let batchBytes = 0 + for (const file of files) { + for await (const { stream, payload } of readDockerLogFrames( + createReadStream(file) + )) { + const { ms, body } = splitLogTimestamp(payload) + if (sinceMs !== undefined && ms !== undefined && ms < sinceMs) continue + const frame = encodeDockerLogFrame(stream, body) + batch.push(frame) + batchBytes += frame.length + if (batchBytes >= 64 * 1024) { + yield Buffer.concat(batch) + batch = [] + batchBytes = 0 + } + } + } + if (batch.length) yield Buffer.concat(batch) + } + return Readable.from(chunks(), { objectMode: false }) +} diff --git a/src/components/core/service/getStreamableLogs.ts b/src/components/core/service/getStreamableLogs.ts index fb6ebcfdf..4326a6a3e 100644 --- a/src/components/core/service/getStreamableLogs.ts +++ b/src/components/core/service/getStreamableLogs.ts @@ -78,7 +78,10 @@ export class ServiceGetStreamableLogsHandler extends CommandHandler { if (!respStream) { return { stream: null, - status: { httpStatus: 404, error: 'Service not found or not running' } + status: { + httpStatus: 404, + error: `Service ${task.serviceId} has no container and no saved logs` + } } } return { stream: respStream as unknown as Stream, status: { httpStatus: 200 } } diff --git a/src/components/core/service/utils.ts b/src/components/core/service/utils.ts index 739b4719f..c069a3e06 100644 --- a/src/components/core/service/utils.ts +++ b/src/components/core/service/utils.ts @@ -74,7 +74,7 @@ export function toPublicServiceJob( // Listing-grade sanitization for SERVICE_LIST, which is NOT owner-scoped: on top of the // always-stripped userData (the encrypted env blob), it removes everything that reveals // HOW a service is configured — CMD/ENTRYPOINT overrides and any inline Dockerfile — and -// what it produced (its output archives), keeping identity, status, resources, endpoints, payment metadata and the owner's +// what it produced (its output archives and saved logs), keeping identity, status, resources, endpoints, payment metadata and the owner's // arbitrary `metadata` labels. `metadata` is returned in both this node-wide list and the // owner-scoped SERVICE_GET_STATUS. export function toListedServiceJob( @@ -88,6 +88,7 @@ export function toListedServiceJob( | 'dockerfile' | 'additionalDockerFiles' | 'outputArchives' + | 'logArchives' | 'previousContainerId' > | null { if (!job) return null @@ -99,6 +100,7 @@ export function toListedServiceJob( dockerfile, additionalDockerFiles, outputArchives, + logArchives, previousContainerId, terminationReason, readiness, diff --git a/src/components/database/sqliteCompute.ts b/src/components/database/sqliteCompute.ts index 900cde15b..cc1aa422f 100644 --- a/src/components/database/sqliteCompute.ts +++ b/src/components/database/sqliteCompute.ts @@ -520,10 +520,10 @@ export class SQLiteCompute implements ComputeDatabaseProvider { } // Expired (terminal) services whose paid window ended at or before `expiresBefore` (Unix - // ms) and that still hold output archives — candidates for deleting them once storage - // expiry has elapsed. The archive filter runs in SQL: expired rows are kept forever (with - // outputArchives: [] once cleaned), so without it every cleanup run would load and parse - // every service that ever expired. + // ms) and that still hold output archives or saved logs — candidates for deleting them once + // storage expiry has elapsed. The filter runs in SQL: expired rows are kept forever (with + // outputArchives: [] and logArchives: [] once cleaned), so without it every cleanup run would + // load and parse every service that ever expired. // eslint-disable-next-line require-await async getExpiredServiceJobsBefore( expiresBefore: number, @@ -532,7 +532,8 @@ export class SQLiteCompute implements ComputeDatabaseProvider { const params: Array = [ServiceStatusNumber.Expired, expiresBefore] let selectSQL = `SELECT * FROM service_jobs WHERE status = ? AND expiresAt <= ?` + - ` AND json_array_length(CAST(body AS TEXT), '$.outputArchives') > 0` + ` AND (json_array_length(CAST(body AS TEXT), '$.outputArchives') > 0` + + ` OR json_array_length(CAST(body AS TEXT), '$.logArchives') > 0)` if (clusterHash) { selectSQL += ` AND clusterHash = ?` params.push(clusterHash) diff --git a/src/test/integration/services.test.ts b/src/test/integration/services.test.ts index 7806788f7..8a83cd8db 100644 --- a/src/test/integration/services.test.ts +++ b/src/test/integration/services.test.ts @@ -1486,6 +1486,33 @@ describe('********** Service on Demand', () => { expect((await getServiceResult({ live: true })).status.httpStatus).to.equal(409) }) + it('(m3) SERVICE_GET_STREAMABLE_LOGS still serves the stopped service logs, saved at teardown', async function () { + const job = await getServiceJob(serviceId) + expect( + job.logArchives, + 'stop must save the container logs' + ).to.have.length.greaterThan(0) + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS) + const resp = await new ServiceGetStreamableLogsHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS, + consumerAddress: addr, + nonce, + signature, + serviceId + } as ServiceGetStreamableLogsCommand) + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + // nginx logs its startup banner, so the last container's saved log is not empty + const logs = await readLogsWithTimeout(resp.stream as Readable) + expect(logs.length).to.be.greaterThan(0) + }) + it('(n) [slow] expiry cron marks a short-lived service Expired', async function () { this.timeout(150000) const { diff --git a/src/test/unit/service/serviceHandlers.test.ts b/src/test/unit/service/serviceHandlers.test.ts index 3bd7cfde1..7b2e0479d 100644 --- a/src/test/unit/service/serviceHandlers.test.ts +++ b/src/test/unit/service/serviceHandlers.test.ts @@ -1433,12 +1433,13 @@ describe('Service handlers', () => { expect(res.status.httpStatus).to.equal(401) }) - it('404 when the engine has no stream to return (not running/error)', async () => { + it('404 when the engine has no stream to return (no container, no saved logs)', async () => { const { node } = buildFakes({ serviceJobInDb: makeJob(), streamableLogs: null }) const res = await new ServiceGetStreamableLogsHandler(node).handle({ ...baseTask } as any) expect(res.status.httpStatus).to.equal(404) + expect(res.status.error).to.contain('no container and no saved logs') }) it('200 and returns the engine stream on success', async () => { diff --git a/src/test/unit/service/serviceJobsDatabase.test.ts b/src/test/unit/service/serviceJobsDatabase.test.ts index 3ae97e754..6843f4824 100644 --- a/src/test/unit/service/serviceJobsDatabase.test.ts +++ b/src/test/unit/service/serviceJobsDatabase.test.ts @@ -440,6 +440,34 @@ describe('Service Jobs Database', () => { ).to.deep.equal([archive]) }) + it('getExpiredServiceJobsBefore also returns Expired jobs that hold only saved logs', async () => { + const logs = { + index: 0, + filename: 'logs-0.log', + filesize: 42, + createdAt: Date.now(), + containerId: 'container-1' + } + const expired = { + status: ServiceStatusNumber.Expired, + statusText: 'Expired', + expiresAt: Date.now() - 10_000 + } + // a service with an output bucket has no output archives, only its saved logs + const onlyLogs = makeServiceJob({ ...expired, logArchives: [logs] }) + const cleaned = makeServiceJob({ ...expired, outputArchives: [], logArchives: [] }) + await db.newServiceJob(onlyLogs) + await db.newServiceJob(cleaned) + + const jobs = await db.getExpiredServiceJobsBefore(Date.now() - 5_000, CLUSTER_HASH) + const ids = jobs.map((j) => j.serviceId) + expect(ids).to.include(onlyLogs.serviceId) + expect(ids).to.not.include(cleaned.serviceId) + expect( + jobs.find((j) => j.serviceId === onlyLogs.serviceId).logArchives + ).to.deep.equal([logs]) + }) + it('DiskQuotaExceeded reserves claimed resources, expires normally, and waits for Expired before archive cleanup', async () => { const diskStopped = makeServiceJob({ status: ServiceStatusNumber.DiskQuotaExceeded, diff --git a/src/test/unit/service/serviceLogsArchive.test.ts b/src/test/unit/service/serviceLogsArchive.test.ts new file mode 100644 index 000000000..b36d3cbf7 --- /dev/null +++ b/src/test/unit/service/serviceLogsArchive.test.ts @@ -0,0 +1,476 @@ +/* eslint-disable security/detect-non-literal-fs-filename */ +import { expect } from 'chai' +import sinon from 'sinon' +import fs from 'fs' +import os from 'os' +import path from 'path' +import { PassThrough, Readable } from 'stream' +import { C2DEngineDocker } from '../../../components/c2d/compute_engine_docker.js' +import { + collectServiceLogs, + encodeDockerLogFrame, + readDockerLogFrames, + savedServiceLogsStream, + splitLogTimestamp +} from '../../../components/c2d/serviceLogs.js' +import { ServiceStatusNumber, ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' + +const OWNER = '0x0000000000000000000000000000000000000001' +const SERVICE_ID = 'svc-logs-1' +const CLUSTER_HASH = 'hash-logs' +const STDOUT = 1 +const STDERR = 2 + +const T0 = '2026-11-17T10:00:00.000000001Z' +const T1 = '2026-11-17T10:00:01.000000001Z' +const T2 = '2026-11-17T10:00:02.000000001Z' +const T3 = '2026-11-17T11:00:00.000000001Z' + +// One entry as `container.logs({ timestamps: true })` sends it. +function entry(at: string, text: string, stream = STDOUT): Buffer { + return encodeDockerLogFrame(stream, Buffer.from(`${at} ${text}\n`)) +} + +const LOG = Buffer.concat([ + entry(T0, 'booting'), + entry(T1, 'warning', STDERR), + entry(T2, 'ready') +]) + +async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { + const chunks: Buffer[] = [] + for await (const c of stream as AsyncIterable) chunks.push(c) + return Buffer.concat(chunks) +} + +// The frames of a multiplexed stream, as [stream type, text]. +async function framesOf( + source: Buffer | NodeJS.ReadableStream +): Promise> { + const input = Buffer.isBuffer(source) ? Readable.from([source]) : source + const frames: Array<[number, string]> = [] + for await (const f of readDockerLogFrames(input as AsyncIterable)) { + frames.push([f.stream, f.payload.toString()]) + } + return frames +} + +function makeJob(overrides: Partial = {}): ServiceJob { + return { + serviceId: SERVICE_ID, + clusterHash: CLUSTER_HASH, + environment: 'env-1', + owner: OWNER, + image: 'img', + tag: 'latest', + containerImage: 'img:latest', + containerId: 'c1', + networkId: '', + status: ServiceStatusNumber.Running, + statusText: 'Running', + dateCreated: new Date(0).toISOString(), + expiresAt: Date.now() + 3600_000, + duration: 3600, + exposedPorts: [], + endpoints: [], + resources: [], + payment: { + chainId: 8996, + token: '0xtoken', + lockTx: '0xl', + claimTx: '0xc', + cancelTx: '', + cost: 5 + }, + ...overrides + } +} + +function dockerError(statusCode: number) { + const e: any = new Error(`docker ${statusCode}`) + e.statusCode = statusCode + return e +} + +// Same pattern as serviceOutputsArchive.test.ts: skip the Docker constructor, keep the prototype. +function makeEngine(tempFolder: string): any { + const engine: any = Object.create(C2DEngineDocker.prototype) + engine.docker = { + getNetwork: sinon.stub().returns({ + inspect: sinon.stub().rejects(dockerError(404)), + remove: sinon.stub().resolves(undefined) + }), + getContainer: sinon.stub() + } + engine.db = { + getServiceJob: sinon.stub().resolves([]), + updateServiceJob: sinon.stub().resolves(1), + getExpiredServiceJobsBefore: sinon.stub().resolves([]), + acquireServiceLock: sinon.stub().resolves(true), + releaseServiceLock: sinon.stub().resolves(undefined), + refreshServiceLocks: sinon.stub().resolves(undefined), + isServiceLocked: sinon.stub().resolves(false) + } + engine.serviceLockHolderId = 'test-holder' + engine.cpuAllocations = new Map() + engine.serviceOpsInFlight = new Set() + engine.serviceOpPromises = new Set() + engine.clusterConfig = { + hash: CLUSTER_HASH, + tempFolder: tempFolder + '/', + connection: { resources: [], serviceOnDemand: {} } + } + engine.stopped = false + return engine +} + +// A container whose log is `log`: a stopped container's follow stream sends it all, then ends. +// It has no /data/outputs, which these tests are not about. +function logsContainer(id: string, log: Buffer = LOG) { + return { + id, + inspect: sinon.stub().resolves({}), + stop: sinon.stub().resolves(undefined), + remove: sinon.stub().resolves(undefined), + getArchive: sinon.stub().rejects(dockerError(404)), + logs: sinon.stub().callsFake(() => Promise.resolve(Readable.from([log]))) + } +} + +describe('service logs saved at teardown', () => { + let tempFolder: string + const folder = () => path.join(tempFolder, CLUSTER_HASH, 'services', SERVICE_ID) + + beforeEach(() => { + tempFolder = fs.mkdtempSync(path.join(os.tmpdir(), 'ocean-svc-logs-')) + }) + afterEach(() => { + sinon.restore() + fs.rmSync(tempFolder, { recursive: true, force: true }) + }) + + describe('helpers', () => { + it('reads frames split across chunks, and refuses a stream that is not multiplexed', async () => { + const chunks: Buffer[] = [] + for (let i = 0; i < LOG.length; i += 5) chunks.push(LOG.subarray(i, i + 5)) + expect(await framesOf(Readable.from(chunks))).to.deep.equal([ + [STDOUT, `${T0} booting\n`], + [STDERR, `${T1} warning\n`], + [STDOUT, `${T2} ready\n`] + ]) + + let error: Error | undefined + try { + await framesOf(Buffer.from('raw output of a TTY container\n')) + } catch (e) { + error = e + } + expect(error?.message).to.equal('not a multiplexed Docker log stream') + }) + + it('splits off the timestamp Docker adds, and leaves a payload without one alone', () => { + const stamped = splitLogTimestamp(Buffer.from(`${T1} hello\n`)) + expect(stamped.ms).to.equal(Date.parse('2026-11-17T10:00:01.000Z')) + expect(stamped.body.toString()).to.equal('hello\n') + + const plain = splitLogTimestamp(Buffer.from('no timestamp here\n')) + expect(plain.ms).to.equal(undefined) + expect(plain.body.toString()).to.equal('no timestamp here\n') + }) + + it('collects a whole log that fits', async () => { + const collected = await collectServiceLogs(Readable.from([LOG])) + expect(collected.data.equals(LOG)).to.equal(true) + expect(collected.droppedBytes).to.equal(0) + expect(collected.cutShort).to.equal(false) + expect((await collectServiceLogs(Readable.from([]))).data.length).to.equal(0) + }) + + it('keeps the newest whole frames of a log over the limit, with a note on what was dropped', async () => { + const first = entry(T0, 'booting') + const collected = await collectServiceLogs(Readable.from([LOG]), { + maxBytes: LOG.length - first.length + }) + expect(collected.droppedBytes).to.equal(first.length) + const frames = await framesOf(collected.data) + expect(frames.slice(1)).to.deep.equal([ + [STDERR, `${T1} warning\n`], + [STDOUT, `${T2} ready\n`] + ]) + // the note goes to stderr, stamped like the first entry it precedes + const [noteStream, note] = frames[0] + expect(noteStream).to.equal(STDERR) + expect(note).to.match( + /^2026-11-17T10:00:01\.000Z \[ocean-node\] \d+ bytes of older output/ + ) + expect(note).to.contain(`${first.length} bytes`) + }) + + it('gives up on a log that never ends, keeping what it read', async () => { + const live = new PassThrough() + live.write(entry(T0, 'still running')) + const collected = await collectServiceLogs(live, { idleMs: 50, maxMs: 1000 }) + expect(collected.cutShort).to.equal(true) + expect(await framesOf(collected.data)).to.deep.equal([ + [STDOUT, `${T0} still running\n`] + ]) + expect(live.destroyed).to.equal(true) + }) + + it('streams saved files oldest first, without timestamps, from `since` on', async () => { + fs.mkdirSync(folder(), { recursive: true }) + const a = path.join(folder(), 'logs-0.log') + const b = path.join(folder(), 'logs-1.log') + fs.writeFileSync(a, LOG) + fs.writeFileSync(b, entry(T3, 'second container')) + + expect(await framesOf(savedServiceLogsStream([a, b]))).to.deep.equal([ + [STDOUT, 'booting\n'], + [STDERR, 'warning\n'], + [STDOUT, 'ready\n'], + [STDOUT, 'second container\n'] + ]) + // Unix seconds, like Docker's filter: an entry at or after it is kept + const since = Date.parse('2026-11-17T10:00:01Z') / 1000 + expect(await framesOf(savedServiceLogsStream([a, b], since))).to.deep.equal([ + [STDERR, 'warning\n'], + [STDOUT, 'ready\n'], + [STDOUT, 'second container\n'] + ]) + }) + }) + + describe('saving', () => { + it('stop saves the logs after stopping and before removing the container', async () => { + const engine = makeEngine(tempFolder) + engine.db.getServiceJob.resolves([makeJob()]) + const c = logsContainer('c1') + engine.docker.getContainer.returns(c) + + const stopped = await engine.stopService(SERVICE_ID, OWNER) + + expect(stopped.status).to.equal(ServiceStatusNumber.Stopped) + sinon.assert.callOrder(c.stop, c.logs, c.remove) + sinon.assert.calledWithMatch(c.logs, { + stdout: true, + stderr: true, + follow: true, + timestamps: true + }) + expect(stopped.logArchives).to.have.lengthOf(1) + const [saved] = stopped.logArchives + expect(saved).to.include({ index: 0, filename: 'logs-0.log', containerId: 'c1' }) + expect(saved).to.not.have.property('droppedBytes') + const file = path.join(folder(), 'logs-0.log') + expect(fs.readFileSync(file).equals(LOG)).to.equal(true) + expect(saved.filesize).to.equal(LOG.length) + expect(fs.existsSync(file + '.partial')).to.equal(false) + // persisted with the Stopped job + sinon.assert.calledWithMatch(engine.db.updateServiceJob, { + status: ServiceStatusNumber.Stopped, + logArchives: stopped.logArchives + }) + }) + + it('the expiry teardown saves the logs, for a service with an output bucket too', async () => { + const engine = makeEngine(tempFolder) + engine.db.getServiceJob.resolves([ + makeJob({ expiresAt: Date.now() - 1000, outputBucketId: 'bucket-1' }) + ]) + engine.docker.getContainer.returns(logsContainer('c1')) + const stopped = await engine.stopService(SERVICE_ID, OWNER, true) + expect(stopped.logArchives.map((a: any) => a.containerId)).to.deep.equal(['c1']) + expect(stopped.outputArchives).to.equal(undefined) + }) + + it('numbers saved logs per container and never saves the same container twice', async () => { + const engine = makeEngine(tempFolder) + const job = makeJob() + await engine.saveServiceLogs(job, logsContainer('c1')) + await engine.saveServiceLogs(job, logsContainer('c1')) // retried teardown + await engine.saveServiceLogs(job, logsContainer('c2')) + expect(job.logArchives.map((a) => [a.index, a.containerId])).to.deep.equal([ + [0, 'c1'], + [1, 'c2'] + ]) + expect(fs.readdirSync(folder()).sort()).to.deep.equal(['logs-0.log', 'logs-1.log']) + }) + + it('saves nothing for a container that logged nothing, or is already gone', async () => { + const engine = makeEngine(tempFolder) + const job = makeJob() + await engine.saveServiceLogs(job, logsContainer('c1', Buffer.alloc(0))) + const gone = logsContainer('c2') + gone.logs = sinon.stub().rejects(dockerError(404)) + await engine.saveServiceLogs(job, gone) + expect(job.logArchives).to.equal(undefined) + expect(fs.existsSync(folder())).to.equal(false) + }) + + it('logs a failure without throwing, and the teardown goes on', async () => { + const engine = makeEngine(tempFolder) + engine.db.getServiceJob.resolves([makeJob()]) + const c = logsContainer('c1', Buffer.from('not a multiplexed stream')) + engine.docker.getContainer.returns(c) + const stopped = await engine.stopService(SERVICE_ID, OWNER) + expect(stopped.status).to.equal(ServiceStatusNumber.Stopped) + expect(stopped.logArchives).to.equal(undefined) + sinon.assert.calledOnce(c.remove) + expect(fs.existsSync(folder())).to.equal(false) + }) + + it('restart saves the old container logs before removing it', async () => { + const old = logsContainer('old') + const created = { + id: 'new', + infoArchive: sinon.stub().resolves({}), + putArchive: sinon.stub().resolves(undefined), + start: sinon.stub().resolves(undefined), + stop: sinon.stub().resolves(undefined), + remove: sinon.stub().resolves(undefined) + } + const engine = makeEngine(tempFolder) + engine.docker.getContainer.callsFake((id: string) => + id === created.id ? created : old + ) + engine.docker.createNetwork = sinon.stub().resolves({ id: 'newnet' }) + engine.docker.createContainer = sinon.stub().resolves(created) + engine.pullImageRef = sinon.stub().resolves(undefined) + const job = makeJob({ containerId: 'old' }) + + await engine.doRestartService(job) + + expect(job.status).to.equal(ServiceStatusNumber.Running) + expect(job.containerId).to.equal('new') + expect(job.logArchives.map((a) => a.containerId)).to.deep.equal(['old']) + sinon.assert.callOrder(old.stop, old.logs, old.remove, created.start) + }) + }) + + describe('getServiceStreamableLogs', () => { + // A job whose containers c1 and c2 are gone, their logs saved. + async function withSavedLogs(overrides: Partial = {}) { + const engine = makeEngine(tempFolder) + const job = makeJob({ + status: ServiceStatusNumber.Stopped, + containerId: '', + ...overrides + }) + await engine.saveServiceLogs(job, logsContainer('c1')) + await engine.saveServiceLogs( + job, + logsContainer('c2', entry(T3, 'second container')) + ) + engine.db.getServiceJob.resolves([job]) + return { engine, job } + } + + it('streams the live log while the service has a container', async () => { + const engine = makeEngine(tempFolder) + engine.db.getServiceJob.resolves([makeJob()]) + const c = logsContainer('c1') + engine.docker.getContainer.returns(c) + + const stream = await engine.getServiceStreamableLogs(SERVICE_ID, OWNER, 1234) + + sinon.assert.calledWithMatch(c.logs, { follow: true, since: 1234 }) + expect(c.logs.firstCall.args[0]).to.not.have.property('timestamps') + expect((await streamToBuffer(stream)).equals(LOG)).to.equal(true) + }) + + it('serves the saved logs once the containers are gone, oldest first, as a live stream would', async () => { + const { engine } = await withSavedLogs() + expect( + await framesOf(await engine.getServiceStreamableLogs(SERVICE_ID, OWNER)) + ).to.deep.equal([ + [STDOUT, 'booting\n'], + [STDERR, 'warning\n'], + [STDOUT, 'ready\n'], + [STDOUT, 'second container\n'] + ]) + const since = Date.parse('2026-11-17T11:00:00Z') / 1000 + expect( + await framesOf(await engine.getServiceStreamableLogs(SERVICE_ID, OWNER, since)) + ).to.deep.equal([[STDOUT, 'second container\n']]) + }) + + it('falls back to the saved logs when the container of an errored service is gone', async () => { + const { engine } = await withSavedLogs({ + status: ServiceStatusNumber.Error, + containerId: 'c3' + }) + engine.docker.getContainer.returns({ + inspect: sinon.stub().rejects(dockerError(404)) + }) + const frames = await framesOf( + await engine.getServiceStreamableLogs(SERVICE_ID, OWNER) + ) + expect(frames).to.have.lengthOf(4) + }) + + it('returns null on a Docker failure rather than serving older logs in its place', async () => { + // running again after a restart, with the saved logs of its earlier containers + const { engine } = await withSavedLogs({ + status: ServiceStatusNumber.Running, + containerId: 'c3' + }) + engine.docker.getContainer.returns({ + inspect: sinon.stub().rejects(dockerError(500)) + }) + expect(await engine.getServiceStreamableLogs(SERVICE_ID, OWNER)).to.equal(null) + }) + + it('returns null when nothing was saved, or the files were deleted', async () => { + const engine = makeEngine(tempFolder) + expect(await engine.getServiceStreamableLogs(SERVICE_ID, OWNER)).to.equal(null) + engine.db.getServiceJob.resolves([ + makeJob({ status: ServiceStatusNumber.Stopped, containerId: '' }) + ]) + expect(await engine.getServiceStreamableLogs(SERVICE_ID, OWNER)).to.equal(null) + + const saved = await withSavedLogs() + fs.rmSync(folder(), { recursive: true, force: true }) + expect(await saved.engine.getServiceStreamableLogs(SERVICE_ID, OWNER)).to.equal( + null + ) + }) + }) + + describe('cleanupExpiredServiceOutputs', () => { + const STORAGE_EXPIRY = 3600 // seconds + + async function expiredService(expiredAgo: number) { + const engine = makeEngine(tempFolder) + engine.getComputeEnvironments = sinon + .stub() + .resolves([{ id: 'env-1', storageExpiry: STORAGE_EXPIRY }]) + // an output bucket: the saved logs are all the node holds for it + const job = makeJob({ + status: ServiceStatusNumber.Expired, + containerId: '', + outputBucketId: 'bucket-1', + expiresAt: Date.now() - expiredAgo + }) + await engine.saveServiceLogs(job, logsContainer('c1')) + engine.db.getServiceJob.resolves([job]) + engine.db.getExpiredServiceJobsBefore.resolves([job]) + return { engine, job } + } + + it('deletes the saved logs once storageExpiry has elapsed since expiresAt', async () => { + const { engine, job } = await expiredService(STORAGE_EXPIRY * 1000 + 1000) + expect(await engine.cleanupExpiredServiceOutputs()).to.equal(1) + expect(job.logArchives).to.deep.equal([]) + expect(fs.existsSync(folder())).to.equal(false) + sinon.assert.calledWith(engine.db.updateServiceJob, job) + expect(await engine.getServiceStreamableLogs(SERVICE_ID, OWNER)).to.equal(null) + }) + + it('keeps them until then', async () => { + const { engine, job } = await expiredService(STORAGE_EXPIRY * 1000 - 60_000) + expect(await engine.cleanupExpiredServiceOutputs()).to.equal(0) + expect(job.logArchives).to.have.lengthOf(1) + sinon.assert.notCalled(engine.db.updateServiceJob) + }) + }) +}) diff --git a/src/test/unit/service/serviceOutputsArchive.test.ts b/src/test/unit/service/serviceOutputsArchive.test.ts index 266f401ce..17ae1a3fb 100644 --- a/src/test/unit/service/serviceOutputsArchive.test.ts +++ b/src/test/unit/service/serviceOutputsArchive.test.ts @@ -167,7 +167,9 @@ function outputsContainer(id: string, entries: TarEntry[] = OUTPUTS_TAR) { id, stop: sinon.stub().resolves(undefined), remove: sinon.stub().resolves(undefined), - getArchive: sinon.stub().callsFake(async () => Readable.from(await makeTar(entries))) + getArchive: sinon.stub().callsFake(async () => Readable.from(await makeTar(entries))), + // teardown also saves the container's logs (serviceLogsArchive.test.ts); these log nothing + logs: sinon.stub().callsFake(() => Promise.resolve(Readable.from([]))) } } diff --git a/src/test/unit/service/serviceUtils.test.ts b/src/test/unit/service/serviceUtils.test.ts index 4a501b0af..f807b7415 100644 --- a/src/test/unit/service/serviceUtils.test.ts +++ b/src/test/unit/service/serviceUtils.test.ts @@ -77,6 +77,26 @@ describe('service utils', () => { expect(listed).to.not.have.property('dockerfile') expect(listed).to.have.property('serviceId', 's1') }) + it('leaves out what the service produced: output archives and saved logs', () => { + const archive = { + index: 0, + filesize: 1, + createdAt: 0, + containerId: 'c1' + } + const job = { + serviceId: 's1', + owner: '0xabc', + endpoints: [], + outputArchives: [{ ...archive, filename: 'outputs-0.zip' }], + logArchives: [{ ...archive, filename: 'logs-0.log' }] + } as unknown as ServiceJob + const listed = toListedServiceJob(job) + expect(listed).to.not.have.property('outputArchives') + expect(listed).to.not.have.property('logArchives') + // the owner's own status keeps them + expect(toPublicServiceJob(job)).to.have.property('logArchives') + }) it('is null-safe', () => { expect(toListedServiceJob(null)).to.equal(null) }) From a5d12f79089072981071c1b8da0a963e3032ce61 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Wed, 18 Nov 2026 10:14:19 +0200 Subject: [PATCH 2/3] updated comments to make more sens --- src/@types/C2D/ServiceOnDemand.ts | 11 ++-- src/components/c2d/compute_engine_base.ts | 5 +- src/components/c2d/compute_engine_docker.ts | 31 ++++++----- src/components/c2d/serviceLogs.ts | 53 ++++++++++--------- .../unit/service/serviceJobsDatabase.test.ts | 3 +- .../unit/service/serviceLogsArchive.test.ts | 23 ++++---- .../service/serviceOutputsArchive.test.ts | 3 +- src/test/unit/service/serviceUtils.test.ts | 2 +- 8 files changed, 71 insertions(+), 60 deletions(-) diff --git a/src/@types/C2D/ServiceOnDemand.ts b/src/@types/C2D/ServiceOnDemand.ts index 2f9733683..77e7ec47d 100644 --- a/src/@types/C2D/ServiceOnDemand.ts +++ b/src/@types/C2D/ServiceOnDemand.ts @@ -201,16 +201,16 @@ export interface ServiceOutputArchive { containerId: string // container it was taken from — a retried teardown must not archive it twice } -// The stdout/stderr of one container of the service, saved right before it was removed (Docker -// drops a container's logs with it). SERVICE_GET_STREAMABLE_LOGS serves them, oldest first, once -// the service has no container, until expiresAt + the environment's storageExpiry. +// Docker deletes a container's logs together with the container, so each service container's +// stdout/stderr is saved before it is removed. That keeps the logs readable through +// SERVICE_GET_STREAMABLE_LOGS after a stop or expiry, until expiresAt + storageExpiry. export interface ServiceLogArchive { index: number filename: string // e.g. logs-0.log, under /services// filesize: number // bytes createdAt: number // Unix ms timestamp containerId: string // container it was taken from — a retried teardown must not save it twice - // Bytes of the oldest output left out because the container logged more than the node keeps. + // Recorded so the owner can tell a trimmed log from a complete one. droppedBytes?: number } @@ -317,8 +317,7 @@ export interface ServiceJob { outputBucketId?: string // persistent-storage bucket bind-mounted at /data/outputs // Archives of /data/outputs, only for services without an outputBucketId. outputArchives?: ServiceOutputArchive[] - // Saved logs of the containers that are gone, for every service (an output bucket holds only - // /data/outputs). + // Kept for services with an outputBucketId too: the bucket holds only /data/outputs. logArchives?: ServiceLogArchive[] // Internal, only while a restart is in flight: the stopped old container, kept until its // /data/outputs has been carried over into the new one (orphan recovery archives and diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index 5e7ab6231..546ad2aa9 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -209,8 +209,9 @@ export abstract class C2DEngine { return [] } - // The live log of the service's container while it has one, else the logs saved when its - // containers were removed (see ServiceJob.logArchives). Null when there is neither. + // Owners need a service's logs after it stops too, and Docker deletes them with the container: + // so once there is no container, the logs saved at teardown (ServiceJob.logArchives) are served. + // Null when there is neither. // eslint-disable-next-line require-await public async getServiceStreamableLogs( serviceId: string, diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 75a127ba7..db7c33f62 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -4616,12 +4616,11 @@ export class C2DEngineDocker extends C2DEngine { } } - // Saves the container's stdout/stderr to /services//logs-.log and records - // it on job.logArchives, so SERVICE_GET_STREAMABLE_LOGS can serve it once the container is gone: - // Docker drops a container's logs with the container. Must run after the container was stopped - // (its log is then final, and reading it ends by itself) and before it is removed. Unlike the - // outputs it runs for services with an output bucket too. Keeps the newest - // SERVICE_LOGS_MAX_BYTES. Best-effort — never throws, a failure is only logged; the caller + // Docker deletes a container's logs together with the container, so they are saved to the + // service's folder first, for SERVICE_GET_STREAMABLE_LOGS to serve after a stop or expiry. + // Call it after stopping the container, so its log is final and the follow read ends on its + // own, and before removing it. Bucket services need it too: the bucket holds only + // /data/outputs. Never throws, because losing the logs must not block a teardown; the caller // persists the job. private async saveServiceLogs( job: ServiceJob, @@ -6587,7 +6586,8 @@ export class C2DEngineDocker extends C2DEngine { if (!isBenignDockerError(e)) stopped = false logDockerCleanupError(e, `restart ${serviceId}: old container stop: ${e.message}`) }) - // Its logs end here, whichever way it goes below: the new container starts a fresh log. + // Saved here rather than in each branch below: every branch ends with this container + // removed, and the new one starts with an empty log. await this.saveServiceLogs(job, c) if (job.outputBucketId) { await c.remove({ force: true, v: true }).catch((e) => { @@ -6883,8 +6883,9 @@ export class C2DEngineDocker extends C2DEngine { ): Promise { const [job] = await this.db.getServiceJob(serviceId, owner) if (!job) return null - // While the service has a container (Running, or Error before teardown) its live log is the - // answer: it covers everything that container logged. + // The live log wins while a container exists (Running, or Error before teardown): it holds + // everything that container logged and keeps following it, while the saved files only cover + // containers already removed. if ( job.containerId && (job.status === ServiceStatusNumber.Running || @@ -6900,8 +6901,8 @@ export class C2DEngineDocker extends C2DEngine { ...(since !== undefined ? { since } : {}) })) as unknown as NodeJS.ReadableStream } catch (e: any) { - // Gone: fall back to the saved logs. Anything else is a Docker failure, and serving - // older saved logs in place of the live ones would hide it. + // Only a 404 (container gone) falls back to the saved logs. Anything else is a Docker + // failure, and answering it with older saved logs would hide it. if (e?.statusCode !== 404) { CORE_LOGGER.error( `getServiceStreamableLogs failed for ${serviceId}: ${e?.message ?? e}` @@ -6913,8 +6914,9 @@ export class C2DEngineDocker extends C2DEngine { return this.openSavedServiceLogs(job, since) } - // The logs saved at teardown (job.logArchives), oldest container first, or null when there - // are none — never saved, or deleted with the output archives once storageExpiry elapsed. + // Sorted by index so the log reads in the order the containers ran. A file missing on disk is + // skipped rather than failing the stream: if a cleanup deleted the folder but could not persist + // the job, logArchives still lists it. private openSavedServiceLogs( job: ServiceJob, since?: number @@ -7019,7 +7021,8 @@ export class C2DEngineDocker extends C2DEngine { now - shortest * 1000, this.getC2DConfig().hash ) - // The output archives and the saved logs share the service's folder and expire together. + // Either kind of file keeps a service a candidate: both live in the service's folder, which + // is deleted as a whole. const holdsFiles = (job?: ServiceJob) => !!(job?.outputArchives?.length || job?.logArchives?.length) let cleaned = 0 diff --git a/src/components/c2d/serviceLogs.ts b/src/components/c2d/serviceLogs.ts index 416a9e404..372d947be 100644 --- a/src/components/c2d/serviceLogs.ts +++ b/src/components/c2d/serviceLogs.ts @@ -1,19 +1,20 @@ import { createReadStream } from 'fs' import { Readable } from 'stream' -// The most of one container's log the node saves at teardown. A service can run for weeks, and -// Docker keeps everything it logged, so only the most recent output is kept: that is where a -// crash or a stop shows up. +// A service can run for weeks and Docker keeps everything it logged, so saving all of it would let +// one chatty container take unbounded memory at teardown and disk until expiry. The newest output +// is what's kept, because that is where a crash or a stop shows up. export const SERVICE_LOGS_MAX_BYTES = 16 * 1024 * 1024 -// Reading a stopped container's log ends by itself once Docker has sent it all. These bound the -// read when the container could not be stopped and is still logging. +// Reading a stopped container's log ends by itself. These exist for a container that could not be +// stopped: its log never ends, and a teardown must not wait on it forever. export const SERVICE_LOGS_IDLE_MS = 5_000 export const SERVICE_LOGS_MAX_MS = 60_000 -// A non-TTY container's stdout and stderr come multiplexed: each write is a frame with an 8-byte -// header — [stream type, 0, 0, 0, payload size as uint32 BE] — followed by the payload. Service -// containers never get a TTY, so their logs always have this shape. +// Saved files keep Docker's multiplexed framing rather than plain text, so they stream back in the +// same shape as a live log and clients need no change. Each frame is an 8-byte header — +// [stream type, 0, 0, 0, payload size as uint32 BE] — and the payload. Service containers never get +// a TTY, so their logs are always framed this way. const FRAME_HEADER_BYTES = 8 const STDERR = 2 @@ -30,9 +31,9 @@ export function encodeDockerLogFrame(stream: number, payload: Buffer): Buffer { } /** - * Yields the frames of a multiplexed Docker log stream. Throws on bytes that are not a frame - * header (e.g. the raw output of a TTY container). A frame cut off by the end of the stream is - * dropped. + * Throws on bytes that aren't a frame header (e.g. a TTY container's raw output) rather than + * guessing: a misread size would garble everything after it. A frame cut off by the end of the + * stream (a read that gave up mid-frame) is dropped, as its header claims bytes that never came. */ export async function* readDockerLogFrames( source: AsyncIterable @@ -63,12 +64,14 @@ export async function* readDockerLogFrames( } /** - * Splits off the RFC 3339 timestamp Docker puts in front of each entry when asked for - * `timestamps: true`. `ms` is undefined for a payload without one. + * Logs are saved with Docker's per-entry timestamps (`timestamps: true`) only so `since` can be + * applied when serving them; this separates the time from the text the container wrote. + * `ms` is undefined for a payload without one. */ export function splitLogTimestamp(payload: Buffer): { ms?: number; body: Buffer } { const space = payload.indexOf(0x20) - // "2026-11-17T10:00:00.123456789Z" is 30 characters, a numeric offset makes it 35. + // Bounds around "2026-11-17T10:00:00.123456789Z" (30 characters, 35 with a numeric offset), so + // a line that merely starts with a word and a space isn't handed to Date.parse. if (space >= 20 && space <= 40) { const ms = Date.parse(payload.toString('latin1', 0, space)) if (!Number.isNaN(ms)) return { ms, body: payload.subarray(space + 1) } @@ -77,18 +80,20 @@ export function splitLogTimestamp(payload: Buffer): { ms?: number; body: Buffer } export interface CollectedServiceLogs { - // The frames to save, still multiplexed and timestamped, or empty if the container logged - // nothing. Starts with a note on stderr when older output was dropped. + // Left multiplexed and timestamped, so it can be written as is and later served like a live log, + // `since` included. Starts with a note on stderr when older output was dropped, so a reader of + // the log knows it is trimmed. Empty if the container logged nothing. data: Buffer droppedBytes: number - // The read was cut short by `idleMs`/`maxMs`, so the newest output may be missing. + // Reported so the caller can warn that the newest output may be missing. cutShort: boolean } /** - * Reads a container's whole log (`container.logs({ follow: true, timestamps: true })`) and keeps - * the newest `maxBytes` of it, dropping whole frames from the start. Holds at most `maxBytes` in - * memory. Gives up after `idleMs` without data or `maxMs` in all, keeping what it read. + * Trims to the newest `maxBytes` while reading rather than at the end, so a weeks-long log never + * sits in memory in full. Drops whole frames, never part of one, so what's left is still valid + * framing. When it gives up (`idleMs`/`maxMs`) it keeps what it read: at teardown there is no + * second chance, and part of a log beats none. */ export async function collectServiceLogs( source: NodeJS.ReadableStream, @@ -117,7 +122,8 @@ export async function collectServiceLogs( // A copy, so the kept frames don't pin the (larger) chunks they were read from. frames.push({ stream, payload: Buffer.from(payload) }) keptBytes += FRAME_HEADER_BYTES + payload.length - // Always keep the newest frame, even one bigger than the limit on its own. + // The newest frame stays even when it alone exceeds the limit: it's the last thing the + // container said, and dropping it would leave nothing. while (keptBytes > maxBytes && head < frames.length - 1) { const size = FRAME_HEADER_BYTES + frames[head].payload.length frames[head] = undefined @@ -153,9 +159,8 @@ export async function collectServiceLogs( } /** - * Streams saved log files, oldest first, in the same multiplexed format as a live - * `container.logs()` stream. With `since` (Unix seconds, like Docker's own filter) older entries - * are left out. The timestamps the files were saved with are stripped, as a live stream has none. + * Matches a live `container.logs()` stream (same framing, no timestamps), so clients read saved + * logs exactly as they read live ones. `since` takes Unix seconds, like Docker's own filter. */ export function savedServiceLogsStream(files: string[], since?: number): Readable { const sinceMs = since === undefined ? undefined : since * 1000 diff --git a/src/test/unit/service/serviceJobsDatabase.test.ts b/src/test/unit/service/serviceJobsDatabase.test.ts index 6843f4824..89b26f6a4 100644 --- a/src/test/unit/service/serviceJobsDatabase.test.ts +++ b/src/test/unit/service/serviceJobsDatabase.test.ts @@ -453,7 +453,8 @@ describe('Service Jobs Database', () => { statusText: 'Expired', expiresAt: Date.now() - 10_000 } - // a service with an output bucket has no output archives, only its saved logs + // like a bucket service, it has no output archives: only the logArchives half of the filter + // can match it const onlyLogs = makeServiceJob({ ...expired, logArchives: [logs] }) const cleaned = makeServiceJob({ ...expired, outputArchives: [], logArchives: [] }) await db.newServiceJob(onlyLogs) diff --git a/src/test/unit/service/serviceLogsArchive.test.ts b/src/test/unit/service/serviceLogsArchive.test.ts index b36d3cbf7..91eaefdef 100644 --- a/src/test/unit/service/serviceLogsArchive.test.ts +++ b/src/test/unit/service/serviceLogsArchive.test.ts @@ -26,7 +26,7 @@ const T1 = '2026-11-17T10:00:01.000000001Z' const T2 = '2026-11-17T10:00:02.000000001Z' const T3 = '2026-11-17T11:00:00.000000001Z' -// One entry as `container.logs({ timestamps: true })` sends it. +// Built by hand in the shape `container.logs({ timestamps: true })` sends, so no Docker is needed. function entry(at: string, text: string, stream = STDOUT): Buffer { return encodeDockerLogFrame(stream, Buffer.from(`${at} ${text}\n`)) } @@ -43,7 +43,7 @@ async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { return Buffer.concat(chunks) } -// The frames of a multiplexed stream, as [stream type, text]. +// [stream type, text] pairs, so a failing assertion shows readable log lines rather than bytes. async function framesOf( source: Buffer | NodeJS.ReadableStream ): Promise> { @@ -92,7 +92,8 @@ function dockerError(statusCode: number) { return e } -// Same pattern as serviceOutputsArchive.test.ts: skip the Docker constructor, keep the prototype. +// Built from the prototype so the constructor, which connects to Docker, never runs (same pattern +// as serviceOutputsArchive.test.ts). function makeEngine(tempFolder: string): any { const engine: any = Object.create(C2DEngineDocker.prototype) engine.docker = { @@ -124,8 +125,8 @@ function makeEngine(tempFolder: string): any { return engine } -// A container whose log is `log`: a stopped container's follow stream sends it all, then ends. -// It has no /data/outputs, which these tests are not about. +// Its log stream ends after `log`, as a stopped container's does, so saving never waits on the +// idle timer. getArchive answers 404 to keep the outputs archive out of these tests. function logsContainer(id: string, log: Buffer = LOG) { return { id, @@ -197,7 +198,7 @@ describe('service logs saved at teardown', () => { [STDERR, `${T1} warning\n`], [STDOUT, `${T2} ready\n`] ]) - // the note goes to stderr, stamped like the first entry it precedes + // stamped like the first entry it precedes, so `since` keeps or drops the two together const [noteStream, note] = frames[0] expect(noteStream).to.equal(STDERR) expect(note).to.match( @@ -230,7 +231,7 @@ describe('service logs saved at teardown', () => { [STDOUT, 'ready\n'], [STDOUT, 'second container\n'] ]) - // Unix seconds, like Docker's filter: an entry at or after it is kept + // Unix seconds, the unit clients already send for Docker's own filter const since = Date.parse('2026-11-17T10:00:01Z') / 1000 expect(await framesOf(savedServiceLogsStream([a, b], since))).to.deep.equal([ [STDERR, 'warning\n'], @@ -265,7 +266,7 @@ describe('service logs saved at teardown', () => { expect(fs.readFileSync(file).equals(LOG)).to.equal(true) expect(saved.filesize).to.equal(LOG.length) expect(fs.existsSync(file + '.partial')).to.equal(false) - // persisted with the Stopped job + // persisted, or the file would sit on disk with no record pointing to it sinon.assert.calledWithMatch(engine.db.updateServiceJob, { status: ServiceStatusNumber.Stopped, logArchives: stopped.logArchives @@ -348,7 +349,7 @@ describe('service logs saved at teardown', () => { }) describe('getServiceStreamableLogs', () => { - // A job whose containers c1 and c2 are gone, their logs saved. + // Two saved containers, so the order they are served in can be checked. async function withSavedLogs(overrides: Partial = {}) { const engine = makeEngine(tempFolder) const job = makeJob({ @@ -409,7 +410,7 @@ describe('service logs saved at teardown', () => { }) it('returns null on a Docker failure rather than serving older logs in its place', async () => { - // running again after a restart, with the saved logs of its earlier containers + // saved logs exist, so a null can only mean the Docker error was not papered over const { engine } = await withSavedLogs({ status: ServiceStatusNumber.Running, containerId: 'c3' @@ -444,7 +445,7 @@ describe('service logs saved at teardown', () => { engine.getComputeEnvironments = sinon .stub() .resolves([{ id: 'env-1', storageExpiry: STORAGE_EXPIRY }]) - // an output bucket: the saved logs are all the node holds for it + // an output bucket leaves no output archives, so only the saved logs can trigger cleanup const job = makeJob({ status: ServiceStatusNumber.Expired, containerId: '', diff --git a/src/test/unit/service/serviceOutputsArchive.test.ts b/src/test/unit/service/serviceOutputsArchive.test.ts index 17ae1a3fb..dc8e1e70f 100644 --- a/src/test/unit/service/serviceOutputsArchive.test.ts +++ b/src/test/unit/service/serviceOutputsArchive.test.ts @@ -168,7 +168,8 @@ function outputsContainer(id: string, entries: TarEntry[] = OUTPUTS_TAR) { stop: sinon.stub().resolves(undefined), remove: sinon.stub().resolves(undefined), getArchive: sinon.stub().callsFake(async () => Readable.from(await makeTar(entries))), - // teardown also saves the container's logs (serviceLogsArchive.test.ts); these log nothing + // teardown reads the logs too (serviceLogsArchive.test.ts); an empty log keeps these tests + // about the outputs alone logs: sinon.stub().callsFake(() => Promise.resolve(Readable.from([]))) } } diff --git a/src/test/unit/service/serviceUtils.test.ts b/src/test/unit/service/serviceUtils.test.ts index f807b7415..31dbf1252 100644 --- a/src/test/unit/service/serviceUtils.test.ts +++ b/src/test/unit/service/serviceUtils.test.ts @@ -94,7 +94,7 @@ describe('service utils', () => { const listed = toListedServiceJob(job) expect(listed).to.not.have.property('outputArchives') expect(listed).to.not.have.property('logArchives') - // the owner's own status keeps them + // stripped from the node-wide list only: the owner still needs them on their own status expect(toPublicServiceJob(job)).to.have.property('logArchives') }) it('is null-safe', () => { From eedd27fb19ee6e931a394dcd10302279edb59a1f Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Fri, 20 Nov 2026 17:22:59 +0200 Subject: [PATCH 3/3] updated comments to make more sens --- docs/API.md | 18 +- docs/services.md | 13 +- src/@types/C2D/ServiceOnDemand.ts | 5 +- src/components/c2d/compute_engine_base.ts | 3 +- src/components/c2d/compute_engine_docker.ts | 73 +++++-- src/components/c2d/serviceLogs.ts | 55 ++++-- src/test/integration/services.test.ts | 35 +++- .../unit/service/serviceLogsArchive.test.ts | 183 ++++++++++++++++-- 8 files changed, 318 insertions(+), 67 deletions(-) diff --git a/docs/API.md b/docs/API.md index abefeeab4..1a1e1fb02 100644 --- a/docs/API.md +++ b/docs/API.md @@ -2974,8 +2974,9 @@ until `expiresAt`. Only `release: true` ends the paid window early. Stream the service container's stdout/stderr logs. **Authenticated and owner-scoped** — only the service's owner (`consumerAddress`, proven by signature/nonce or auth token) can -read its logs. While the service has a container — `Running` (`40`), or `Error` (`99`): a -crashed container is kept around until `stop`/`restart` — its log is followed live. Once the +read its logs. While the service has a container — `Running` (`40`), `Error` (`99`): a +crashed container is kept around until `stop`/`restart`, or `Stopping` until the teardown +removes it — its log is followed live. Once the container is gone, the logs the node saved when each of the service's containers was removed are served instead, oldest first, until `storageExpiry` seconds after `expiresAt`. Only the last 16 MiB of each container's log are kept. @@ -3000,16 +3001,23 @@ logs before reaching the live tail (e.g. `since=1h` for just the last hour). #### Response (400) -`since` is present but not a valid Unix timestamp or duration (``). +`since` is present but not a valid Unix timestamp or duration (``), or no +service with this `serviceId` belongs to `consumerAddress` (`Service job not found`). #### Response (404) -Service not found, or it has no container and no saved logs (never saved, or deleted after +The service exists but has no container and no saved logs (never saved, or deleted after storage expiry). #### Response (401) -Missing/invalid auth, or `consumerAddress` is not the service owner. +Missing/invalid auth. A service owned by another address gets `400`, as above: the lookup is +owner-scoped, so it is not found. + +#### Response (500) + +The node could not read the container's live log from Docker. Saved logs are not served in its +place, so a Docker failure is never mistaken for an empty log. --- diff --git a/docs/services.md b/docs/services.md index 86a8f13de..f47670f55 100644 --- a/docs/services.md +++ b/docs/services.md @@ -40,7 +40,7 @@ and `signature` as query parameters (or an auth-token `Authorization` header). | `SERVICE_RESTART` | `/api/services/serviceRestart` | POST | Recreate the container (no extra charge); asynchronous like start — returns once the job is `Restarting`, poll `serviceStatus`. Optionally restart on a **new image spec** (bug-fix flow) — see below | | `SERVICE_STOP` | `/api/services/serviceStop` | POST | Tear down the container; the paid resource reservation (cpu/ram/gpu + host ports) is kept until `expiresAt`, so the service can be restarted anytime on the same endpoints. `release: true` ends the paid window now and frees it instead (no refund, no restart) | | `SERVICE_GET_TEMPLATES` | `/api/services/serviceTemplates` | GET | List operator-published service templates | -| `SERVICE_GET_STREAMABLE_LOGS` | `/api/services/serviceStreamableLogs` | GET | Stream the container's stdout/stderr logs — authenticated, owner-scoped; followed live while the service has a container (`Running`, or `Error` before teardown), otherwise the logs saved when its containers were removed, until storage expiry; optional `since` to skip history. See [Service logs](#service-logs) | +| `SERVICE_GET_STREAMABLE_LOGS` | `/api/services/serviceStreamableLogs` | GET | Stream the container's stdout/stderr logs — authenticated, owner-scoped; followed live while the service has a container (`Running`, `Error` before teardown, or `Stopping`), otherwise the logs saved when its containers were removed, until storage expiry; optional `since` to skip history. See [Service logs](#service-logs) | | `SERVICE_GET_RESULT` | `/api/services/serviceResult` | GET | Download the service's `/data/outputs` as a zip — one of its archives (`index`, resumable with `offset`) or the running container's (`live=true`); authenticated, owner-scoped. See [Service outputs](#service-outputs) | **Start is asynchronous.** `serviceStart` does only the fast, synchronous validation and then @@ -280,12 +280,15 @@ first, which keeps them available after the session ends: happens for every service, including one with an output bucket, because the bucket only holds `/data/outputs`. Each container gives one file, `logs-.log`, next to the output archives under `/services//`. The files are listed on `serviceStatus` as - `logArchives` (`index`, `filename`, `filesize`, `createdAt`, `containerId`, and - `droppedBytes` when output was cut). Saving is best-effort: if it fails, teardown continues and - the node only logs the error. A container that logged nothing produces no file. + `logArchives` (`index`, `filename`, `filesize`, `createdAt`, `containerId`, `droppedBytes` + when output was cut, and `cutShort` when reading the log timed out, so its newest output may + be missing). Saving is best-effort: if it fails, teardown continues and the node only logs the + error. A container that logged nothing produces no file. - **Only the last 16 MiB of each container's log are kept.** When a container logged more, its oldest output is dropped, and the saved log starts with an `[ocean-node]` line on stderr that - says how much was dropped. + says how much was dropped. The node asks Docker for the last 430,000-odd entries only, which + always add up to more than 16 MiB, so for a longer log the line gives a lower bound ("at + least …"). - **Once the service has no container, `serviceStreamableLogs` serves the saved logs**, oldest container first, in the same multiplexed format as the live stream. The stream ends after the last line instead of following. `since` works the same way. The route returns `404` only when diff --git a/src/@types/C2D/ServiceOnDemand.ts b/src/@types/C2D/ServiceOnDemand.ts index 77e7ec47d..97399c52b 100644 --- a/src/@types/C2D/ServiceOnDemand.ts +++ b/src/@types/C2D/ServiceOnDemand.ts @@ -210,8 +210,11 @@ export interface ServiceLogArchive { filesize: number // bytes createdAt: number // Unix ms timestamp containerId: string // container it was taken from — a retried teardown must not save it twice - // Recorded so the owner can tell a trimmed log from a complete one. + // Recorded so the owner can tell a trimmed log from a complete one. A lower bound for a log + // longer than SERVICE_LOGS_TAIL_ENTRIES entries, since Docker sends no more than that. droppedBytes?: number + // The read timed out before the log ended, so the newest output may be missing. + cutShort?: boolean } export interface ServiceEndpoint { diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index 546ad2aa9..c290ea371 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -211,7 +211,8 @@ export abstract class C2DEngine { // Owners need a service's logs after it stops too, and Docker deletes them with the container: // so once there is no container, the logs saved at teardown (ServiceJob.logArchives) are served. - // Null when there is neither. + // Null when there is neither. Throws when the container can't be read, which is not the same as + // having no logs. // eslint-disable-next-line require-await public async getServiceStreamableLogs( serviceId: string, diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 9e9258cc3..0077b775d 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -124,7 +124,11 @@ import { tarToZip, type ServiceResult } from './serviceOutputsZip.js' -import { collectServiceLogs, savedServiceLogsStream } from './serviceLogs.js' +import { + collectServiceLogs, + savedServiceLogsStream, + SERVICE_LOGS_TAIL_ENTRIES +} from './serviceLogs.js' import { buildServiceHfPlan, getServiceHfMirrorSettings, @@ -4619,8 +4623,9 @@ export class C2DEngineDocker extends C2DEngine { // service's folder first, for SERVICE_GET_STREAMABLE_LOGS to serve after a stop or expiry. // Call it after stopping the container, so its log is final and the follow read ends on its // own, and before removing it. Bucket services need it too: the bucket holds only - // /data/outputs. Never throws, because losing the logs must not block a teardown; the caller - // persists the job. + // /data/outputs. Never throws, because losing the logs must not block a teardown. The caller + // persists the job before removing the container: from then on the file is the only copy, and + // a crash before the record is written would leave it on disk with nothing pointing to it. private async saveServiceLogs( job: ServiceJob, container: Dockerode.Container @@ -4639,7 +4644,8 @@ export class C2DEngineDocker extends C2DEngine { stdout: true, stderr: true, follow: true, - timestamps: true + timestamps: true, + tail: SERVICE_LOGS_TAIL_ENTRIES })) as unknown as NodeJS.ReadableStream } catch (e: any) { if (e?.statusCode === 404) { @@ -4648,7 +4654,9 @@ export class C2DEngineDocker extends C2DEngine { } throw e } - const { data, droppedBytes, cutShort } = await collectServiceLogs(source) + const { data, droppedBytes, cutShort } = await collectServiceLogs(source, { + tail: SERVICE_LOGS_TAIL_ENTRIES + }) if (cutShort) { CORE_LOGGER.warn( `service ${job.serviceId}: reading the logs of container ${container.id} timed out — saving what was read` @@ -4674,7 +4682,8 @@ export class C2DEngineDocker extends C2DEngine { filesize: data.length, createdAt: Date.now(), containerId: container.id, - ...(droppedBytes ? { droppedBytes } : {}) + ...(droppedBytes ? { droppedBytes } : {}), + ...(cutShort ? { cutShort } : {}) } ] CORE_LOGGER.info( @@ -4683,7 +4692,19 @@ export class C2DEngineDocker extends C2DEngine { (droppedBytes ? `, ${droppedBytes} older bytes dropped` : '') ) } catch (e: any) { - if (partialPath) rmSync(partialPath, { force: true }) + // Guarded on its own: rmSync throws on more than a missing file (a permission error, or a + // directory in the way), and escaping from here would stop the teardown this must not block. + if (partialPath) { + try { + rmSync(partialPath, { force: true }) + } catch (cleanupError: any) { + CORE_LOGGER.error( + `service ${job.serviceId}: could not remove ${partialPath}: ${ + cleanupError?.message ?? cleanupError + }` + ) + } + } CORE_LOGGER.error( `service ${job.serviceId}: failed to save the logs of container ${container.id}: ${ e?.message ?? e @@ -4890,6 +4911,8 @@ export class C2DEngineDocker extends C2DEngine { await holder.stop({ t: 10 }).catch(() => {}) await this.saveServiceLogs(job, holder) await this.archiveServiceOutputs(job, holder) + // Recorded before the containers go: a crash after that would otherwise lose the files. + await this.db.updateServiceJob(job) } if (job.containerId) await this.cleanupServiceDocker( @@ -6371,6 +6394,8 @@ export class C2DEngineDocker extends C2DEngine { }) await this.saveServiceLogs(job, c) await this.archiveServiceOutputs(job, c) + // Recorded before the container goes, which takes the only other copy with it. + await this.db.updateServiceJob(job) // v: also drop the anonymous volumes the image declared (e.g. comfyui-boot's // VOLUME /root, where it keeps its models): no later container can attach them, // so without it every stop/restart leaves that data on node disk forever. @@ -6586,8 +6611,9 @@ export class C2DEngineDocker extends C2DEngine { logDockerCleanupError(e, `restart ${serviceId}: old container stop: ${e.message}`) }) // Saved here rather than in each branch below: every branch ends with this container - // removed, and the new one starts with an empty log. + // removed, and the new one starts with an empty log. Recorded before any branch removes it. await this.saveServiceLogs(job, c) + await this.db.updateServiceJob(job) if (job.outputBucketId) { await c.remove({ force: true, v: true }).catch((e) => { logDockerCleanupError( @@ -6808,6 +6834,12 @@ export class C2DEngineDocker extends C2DEngine { }) previous = null delete job.previousContainerId + } else { + // Recorded before it starts, as the branch above does: once running it logs, and a crash + // before step 9 would leave orphan recovery without the id it needs to save that log + // before removing the container. + job.containerId = container.id + await this.db.updateServiceJob(job) } await container.start() @@ -6855,7 +6887,7 @@ export class C2DEngineDocker extends C2DEngine { delete job.previousContainerId } await this.cleanupServiceDocker(container, network, serviceId) - // Set once the carry-over switched to the new container, which is gone now too. + // Set once the new container was recorded (before it started), and it is gone now too. job.containerId = '' // Ports deliberately stay reserved: the consumer paid until expiresAt and may // restart again — the expiry sweep releases them when the window elapses. @@ -6880,15 +6912,16 @@ export class C2DEngineDocker extends C2DEngine { owner: string, since?: number ): Promise { - const [job] = await this.db.getServiceJob(serviceId, owner) + let [job] = await this.db.getServiceJob(serviceId, owner) if (!job) return null - // The live log wins while a container exists (Running, or Error before teardown): it holds - // everything that container logged and keeps following it, while the saved files only cover - // containers already removed. + // The live log wins while a container exists (Running, Error before teardown, or Stopping + // until the teardown removes it): it holds everything that container logged and keeps + // following it, while the saved files only cover containers already removed. if ( job.containerId && (job.status === ServiceStatusNumber.Running || - job.status === ServiceStatusNumber.Error) + job.status === ServiceStatusNumber.Error || + job.status === ServiceStatusNumber.Stopping) ) { try { const container = this.docker.getContainer(job.containerId) @@ -6901,13 +6934,18 @@ export class C2DEngineDocker extends C2DEngine { })) as unknown as NodeJS.ReadableStream } catch (e: any) { // Only a 404 (container gone) falls back to the saved logs. Anything else is a Docker - // failure, and answering it with older saved logs would hide it. + // failure: answering it with older saved logs, or with "no logs", would hide it. The + // details stay in the node's log rather than going to the client. if (e?.statusCode !== 404) { CORE_LOGGER.error( `getServiceStreamableLogs failed for ${serviceId}: ${e?.message ?? e}` ) - return null + throw new Error(`Could not read the logs of service ${serviceId}`) } + // A teardown can save the container's log and remove it after the job was read above, + // so the job is read again for the file it recorded. + const [fresh] = await this.db.getServiceJob(serviceId, owner) + if (fresh) job = fresh } } return this.openSavedServiceLogs(job, since) @@ -6915,7 +6953,8 @@ export class C2DEngineDocker extends C2DEngine { // Sorted by index so the log reads in the order the containers ran. A file missing on disk is // skipped rather than failing the stream: if a cleanup deleted the folder but could not persist - // the job, logArchives still lists it. + // the job, logArchives still lists it. savedServiceLogsStream skips one deleted after this check + // the same way. private openSavedServiceLogs( job: ServiceJob, since?: number diff --git a/src/components/c2d/serviceLogs.ts b/src/components/c2d/serviceLogs.ts index 372d947be..9115b7a80 100644 --- a/src/components/c2d/serviceLogs.ts +++ b/src/components/c2d/serviceLogs.ts @@ -18,6 +18,15 @@ export const SERVICE_LOGS_MAX_MS = 60_000 const FRAME_HEADER_BYTES = 8 const STDERR = 2 +// Docker can be asked for the last N entries of a log, not the last N bytes. Every saved entry is +// at least 39 bytes (the frame header, then a 30-character timestamp and a space), so this many +// always add up to more than SERVICE_LOGS_MAX_BYTES, and the byte limit still decides what is +// kept. Without a tail Docker sends the whole history, weeks of it for a long service, and a read +// that runs out of time before the end loses the newest output, the part worth keeping. +const MIN_SAVED_ENTRY_BYTES = FRAME_HEADER_BYTES + 31 +export const SERVICE_LOGS_TAIL_ENTRIES = + Math.floor(SERVICE_LOGS_MAX_BYTES / MIN_SAVED_ENTRY_BYTES) + 1 + export interface DockerLogFrame { stream: number // 0 stdin, 1 stdout, 2 stderr (3 is Docker's own error stream) payload: Buffer @@ -93,18 +102,22 @@ export interface CollectedServiceLogs { * Trims to the newest `maxBytes` while reading rather than at the end, so a weeks-long log never * sits in memory in full. Drops whole frames, never part of one, so what's left is still valid * framing. When it gives up (`idleMs`/`maxMs`) it keeps what it read: at teardown there is no - * second chance, and part of a log beats none. + * second chance, and part of a log beats none. `tail` is the one the source was requested with: + * a read that reaches it may have had older entries left out by Docker, so the note can then + * only give a lower bound for what was dropped. */ export async function collectServiceLogs( source: NodeJS.ReadableStream, { maxBytes = SERVICE_LOGS_MAX_BYTES, idleMs = SERVICE_LOGS_IDLE_MS, - maxMs = SERVICE_LOGS_MAX_MS - }: { maxBytes?: number; idleMs?: number; maxMs?: number } = {} + maxMs = SERVICE_LOGS_MAX_MS, + tail + }: { maxBytes?: number; idleMs?: number; maxMs?: number; tail?: number } = {} ): Promise { const frames: Array = [] let head = 0 + let entries = 0 let keptBytes = 0 let droppedBytes = 0 let cutShort = false @@ -119,6 +132,7 @@ export async function collectServiceLogs( source as AsyncIterable )) { idleTimer.refresh() + entries++ // A copy, so the kept frames don't pin the (larger) chunks they were read from. frames.push({ stream, payload: Buffer.from(payload) }) keptBytes += FRAME_HEADER_BYTES + payload.length @@ -150,8 +164,9 @@ export async function collectServiceLogs( // Stamped like the first kept entry, so a `since` filter keeps or drops it with that entry. const { ms } = splitLogTimestamp(kept[0].payload) const stamp = ms === undefined ? '' : `${new Date(ms).toISOString()} ` + const atLeast = tail !== undefined && entries >= tail ? 'at least ' : '' const note = - `${stamp}[ocean-node] ${droppedBytes} bytes of older output were dropped: ` + + `${stamp}[ocean-node] ${atLeast}${droppedBytes} bytes of older output were dropped: ` + `the node keeps the last ${maxBytes} bytes of a container's log.\n` parts.unshift(encodeDockerLogFrame(STDERR, Buffer.from(note))) } @@ -168,19 +183,27 @@ export function savedServiceLogsStream(files: string[], since?: number): Readabl let batch: Buffer[] = [] let batchBytes = 0 for (const file of files) { - for await (const { stream, payload } of readDockerLogFrames( - createReadStream(file) - )) { - const { ms, body } = splitLogTimestamp(payload) - if (sinceMs !== undefined && ms !== undefined && ms < sinceMs) continue - const frame = encodeDockerLogFrame(stream, body) - batch.push(frame) - batchBytes += frame.length - if (batchBytes >= 64 * 1024) { - yield Buffer.concat(batch) - batch = [] - batchBytes = 0 + try { + for await (const { stream, payload } of readDockerLogFrames( + createReadStream(file) + )) { + const { ms, body } = splitLogTimestamp(payload) + if (sinceMs !== undefined && ms !== undefined && ms < sinceMs) continue + const frame = encodeDockerLogFrame(stream, body) + batch.push(frame) + batchBytes += frame.length + if (batchBytes >= 64 * 1024) { + yield Buffer.concat(batch) + batch = [] + batchBytes = 0 + } } + } catch (e: any) { + // A file is opened only when its turn comes, after the response has started. If the + // storage-expiry cleanup deleted it in between, it is skipped like a file that was already + // missing, rather than breaking a stream the client was told succeeded. Only the open can + // fail this way: a file deleted while open is still read to the end. + if (e?.code !== 'ENOENT') throw e } } if (batch.length) yield Buffer.concat(batch) diff --git a/src/test/integration/services.test.ts b/src/test/integration/services.test.ts index 8a83cd8db..ceaef57b7 100644 --- a/src/test/integration/services.test.ts +++ b/src/test/integration/services.test.ts @@ -352,6 +352,32 @@ describe('********** Service on Demand', () => { }) } + // Saved logs, unlike a live log, end on their own once the last file is sent. So anything short + // of a clean end fails: an error, or no end within `timeoutMs`, rather than passing on whatever + // arrived first. + // eslint-disable-next-line require-await + async function readEndedLogs(stream: Readable, timeoutMs = 5000): Promise { + return new Promise((resolve, reject) => { + let data = '' + const timer = setTimeout(() => { + stream.removeAllListeners() + stream.destroy() + reject(new Error(`log stream did not end within ${timeoutMs}ms`)) + }, timeoutMs) + stream.on('data', (chunk) => { + data += chunk.toString() + }) + stream.on('end', () => { + clearTimeout(timer) + resolve(data) + }) + stream.on('error', (e) => { + clearTimeout(timer) + reject(e) + }) + }) + } + // ── setup / teardown ───────────────────────────────────────────────── before(async function () { @@ -1508,9 +1534,12 @@ describe('********** Service on Demand', () => { resp.status.httpStatus === 200, `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` ) - // nginx logs its startup banner, so the last container's saved log is not empty - const logs = await readLogsWithTimeout(resp.stream as Readable) - expect(logs.length).to.be.greaterThan(0) + // The image's entrypoint prints this on every boot, so it is in each saved container's log, + // the stopped one included. + const logs = await readEndedLogs(resp.stream as Readable) + expect(logs).to.include( + '/docker-entrypoint.sh: Configuration complete; ready for start up' + ) }) it('(n) [slow] expiry cron marks a short-lived service Expired', async function () { diff --git a/src/test/unit/service/serviceLogsArchive.test.ts b/src/test/unit/service/serviceLogsArchive.test.ts index 91eaefdef..011d7b8ac 100644 --- a/src/test/unit/service/serviceLogsArchive.test.ts +++ b/src/test/unit/service/serviceLogsArchive.test.ts @@ -11,6 +11,8 @@ import { encodeDockerLogFrame, readDockerLogFrames, savedServiceLogsStream, + SERVICE_LOGS_MAX_BYTES, + SERVICE_LOGS_TAIL_ENTRIES, splitLogTimestamp } from '../../../components/c2d/serviceLogs.js' import { ServiceStatusNumber, ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' @@ -138,6 +140,41 @@ function logsContainer(id: string, log: Buffer = LOG) { } } +// The containers whose saved logs had been persisted by the time `removal` was first called. The +// job is one mutable object, so what each update wrote is copied as it happens. +function recordsBefore(engine: any, removal: sinon.SinonStub): () => string[] { + let recorded: string[] = [] + engine.db.updateServiceJob.callsFake((job: ServiceJob) => { + if (!removal.called) recorded = (job.logArchives ?? []).map((a) => a.containerId) + return Promise.resolve(1) + }) + return () => recorded +} + +// A container a restart creates: no log yet, and nothing in /data/outputs. +function createdContainer(id = 'new') { + return { + id, + infoArchive: sinon.stub().resolves({}), + putArchive: sinon.stub().resolves(undefined), + start: sinon.stub().resolves(undefined), + stop: sinon.stub().resolves(undefined), + remove: sinon.stub().resolves(undefined) + } +} + +// An engine whose restart replaces `old` with `created`, with nothing to pull. +function restartEngine(tempFolder: string, old: any, created: any): any { + const engine = makeEngine(tempFolder) + engine.docker.getContainer.callsFake((id: string) => + id === created.id ? created : old + ) + engine.docker.createNetwork = sinon.stub().resolves({ id: 'newnet' }) + engine.docker.createContainer = sinon.stub().resolves(created) + engine.pullImageRef = sinon.stub().resolves(undefined) + return engine +} + describe('service logs saved at teardown', () => { let tempFolder: string const folder = () => path.join(tempFolder, CLUSTER_HASH, 'services', SERVICE_ID) @@ -207,6 +244,22 @@ describe('service logs saved at teardown', () => { expect(note).to.contain(`${first.length} bytes`) }) + it('says "at least" once the read reaches the tail Docker was asked for, as older entries may be missing', async () => { + const first = entry(T0, 'booting') + const trim = { maxBytes: LOG.length - first.length } + // three entries read: below a tail of 4 the count is exact, at a tail of 3 it is a bound + const exact = await collectServiceLogs(Readable.from([LOG]), { ...trim, tail: 4 }) + const bound = await collectServiceLogs(Readable.from([LOG]), { ...trim, tail: 3 }) + const [[, exactNote]] = await framesOf(exact.data) + const [[, boundNote]] = await framesOf(bound.data) + expect(exactNote).to.contain(`[ocean-node] ${first.length} bytes`) + expect(boundNote).to.contain(`[ocean-node] at least ${first.length} bytes`) + // the tail is sized so that the byte limit, not the tail, decides what is kept + expect(SERVICE_LOGS_TAIL_ENTRIES * (8 + T0.length + 1)).to.be.greaterThan( + SERVICE_LOGS_MAX_BYTES + ) + }) + it('gives up on a log that never ends, keeping what it read', async () => { const live = new PassThrough() live.write(entry(T0, 'still running')) @@ -239,6 +292,16 @@ describe('service logs saved at teardown', () => { [STDOUT, 'second container\n'] ]) }) + + it('skips a saved file deleted after the caller checked for it, instead of failing the stream', async () => { + fs.mkdirSync(folder(), { recursive: true }) + const deleted = path.join(folder(), 'logs-0.log') + const kept = path.join(folder(), 'logs-1.log') + fs.writeFileSync(kept, entry(T3, 'second container')) + expect(await framesOf(savedServiceLogsStream([deleted, kept]))).to.deep.equal([ + [STDOUT, 'second container\n'] + ]) + }) }) describe('saving', () => { @@ -247,6 +310,7 @@ describe('service logs saved at teardown', () => { engine.db.getServiceJob.resolves([makeJob()]) const c = logsContainer('c1') engine.docker.getContainer.returns(c) + const recordedBeforeRemoval = recordsBefore(engine, c.remove) const stopped = await engine.stopService(SERVICE_ID, OWNER) @@ -256,8 +320,11 @@ describe('service logs saved at teardown', () => { stdout: true, stderr: true, follow: true, - timestamps: true + timestamps: true, + tail: SERVICE_LOGS_TAIL_ENTRIES }) + // the file must be on record before the container, the only other copy, is removed + expect(recordedBeforeRemoval()).to.deep.equal(['c1']) expect(stopped.logArchives).to.have.lengthOf(1) const [saved] = stopped.logArchives expect(saved).to.include({ index: 0, filename: 'logs-0.log', containerId: 'c1' }) @@ -320,23 +387,27 @@ describe('service logs saved at teardown', () => { expect(fs.existsSync(folder())).to.equal(false) }) + it('a partial file that cannot be removed does not stop the teardown either', async () => { + const engine = makeEngine(tempFolder) + engine.db.getServiceJob.resolves([makeJob()]) + const c = logsContainer('c1') + engine.docker.getContainer.returns(c) + // A directory where the partial file goes: writing it fails, and so does rmSync on it. + fs.mkdirSync(path.join(folder(), 'logs-0.log.partial', 'in-the-way'), { + recursive: true + }) + + const stopped = await engine.stopService(SERVICE_ID, OWNER) + + expect(stopped.status).to.equal(ServiceStatusNumber.Stopped) + expect(stopped.logArchives).to.equal(undefined) + sinon.assert.calledOnce(c.remove) + }) + it('restart saves the old container logs before removing it', async () => { const old = logsContainer('old') - const created = { - id: 'new', - infoArchive: sinon.stub().resolves({}), - putArchive: sinon.stub().resolves(undefined), - start: sinon.stub().resolves(undefined), - stop: sinon.stub().resolves(undefined), - remove: sinon.stub().resolves(undefined) - } - const engine = makeEngine(tempFolder) - engine.docker.getContainer.callsFake((id: string) => - id === created.id ? created : old - ) - engine.docker.createNetwork = sinon.stub().resolves({ id: 'newnet' }) - engine.docker.createContainer = sinon.stub().resolves(created) - engine.pullImageRef = sinon.stub().resolves(undefined) + const created = createdContainer() + const engine = restartEngine(tempFolder, old, created) const job = makeJob({ containerId: 'old' }) await engine.doRestartService(job) @@ -346,6 +417,41 @@ describe('service logs saved at teardown', () => { expect(job.logArchives.map((a) => a.containerId)).to.deep.equal(['old']) sinon.assert.callOrder(old.stop, old.logs, old.remove, created.start) }) + + it('a bucket restart records the old log before removing that container, and the new container before starting it', async () => { + const old = logsContainer('old') + const created = createdContainer() + const engine = restartEngine(tempFolder, old, created) + // What each update wrote, copied as it happens: the job is one mutable object. + const writes: Array<{ + savedLogs: string[] + containerId: string + oldRemoved: boolean + newStarted: boolean + }> = [] + engine.db.updateServiceJob.callsFake((j: ServiceJob) => { + writes.push({ + savedLogs: (j.logArchives ?? []).map((a) => a.containerId), + containerId: j.containerId, + oldRemoved: old.remove.called, + newStarted: created.start.called + }) + return Promise.resolve(1) + }) + // With a bucket there is no carry-over, so the old container is removed straight away. The + // bucket mount itself needs a running node, and is not what this checks. + const job = makeJob({ containerId: 'old', outputBucketId: 'bucket-1' }) + engine.serviceOutputMounts = sinon.stub().resolves([]) + + await engine.doRestartService(job) + + expect(job.status).to.equal(ServiceStatusNumber.Running) + expect(writes.some((w) => !w.oldRemoved && w.savedLogs.includes('old'))).to.equal( + true + ) + // orphan recovery needs the id to save the new container's log if the node dies now + expect(writes.some((w) => !w.newStarted && w.containerId === 'new')).to.equal(true) + }) }) describe('getServiceStreamableLogs', () => { @@ -409,8 +515,40 @@ describe('service logs saved at teardown', () => { expect(frames).to.have.lengthOf(4) }) - it('returns null on a Docker failure rather than serving older logs in its place', async () => { - // saved logs exist, so a null can only mean the Docker error was not papered over + it('streams the live log while a stop is still tearing the container down', async () => { + const engine = makeEngine(tempFolder) + engine.db.getServiceJob.resolves([ + makeJob({ status: ServiceStatusNumber.Stopping, containerId: 'c1' }) + ]) + const c = logsContainer('c1') + engine.docker.getContainer.returns(c) + + const stream = await engine.getServiceStreamableLogs(SERVICE_ID, OWNER) + + sinon.assert.calledOnce(c.logs) + expect((await streamToBuffer(stream)).equals(LOG)).to.equal(true) + }) + + it('reads the job again when the container went mid-request, for the log the teardown just saved', async () => { + const { engine, job } = await withSavedLogs() + // read while the container still existed, before the teardown recorded its log + const stale = makeJob({ status: ServiceStatusNumber.Stopping, containerId: 'c2' }) + engine.db.getServiceJob.onFirstCall().resolves([stale]) + engine.db.getServiceJob.onSecondCall().resolves([job]) + engine.docker.getContainer.returns({ + inspect: sinon.stub().rejects(dockerError(404)) + }) + + const frames = await framesOf( + await engine.getServiceStreamableLogs(SERVICE_ID, OWNER) + ) + + expect(frames).to.have.lengthOf(4) + sinon.assert.calledTwice(engine.db.getServiceJob) + }) + + it('rejects on a Docker failure rather than serving older logs, or none, in its place', async () => { + // saved logs exist, so only the Docker error can be behind the rejection const { engine } = await withSavedLogs({ status: ServiceStatusNumber.Running, containerId: 'c3' @@ -418,7 +556,14 @@ describe('service logs saved at teardown', () => { engine.docker.getContainer.returns({ inspect: sinon.stub().rejects(dockerError(500)) }) - expect(await engine.getServiceStreamableLogs(SERVICE_ID, OWNER)).to.equal(null) + let error: Error | undefined + try { + await engine.getServiceStreamableLogs(SERVICE_ID, OWNER) + } catch (e) { + error = e + } + // generic: Docker's own message stays in the node's log + expect(error?.message).to.equal(`Could not read the logs of service ${SERVICE_ID}`) }) it('returns null when nothing was saved, or the files were deleted', async () => {