diff --git a/src/session/event-log.test.ts b/src/session/event-log.test.ts new file mode 100644 index 000000000..d6ef8a33b --- /dev/null +++ b/src/session/event-log.test.ts @@ -0,0 +1,439 @@ +import { describe, test, expect } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { isSessionId } from "./index.js"; +import { + EVENT_LOG_FILE, + SESSION_EVENT_LOG_ENV, + createEventStreamWriter, + isSessionEventLogEnabled, + openEventStreamWriter, +} from "./event-log.js"; + +function tempDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "event-stream-")); +} + +function rawLines(dir: string, file: string = EVENT_LOG_FILE): string[] { + const raw = fs.readFileSync(path.join(dir, file), "utf8"); + return raw.split("\n").filter((line) => line.length > 0); +} + +describe("createEventStreamWriter", () => { + test("appends one JSON line per event with generated UUIDv7 id, seq, ts, payload", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir, { + now: () => 1_700_000_000_000, + }); + const first = await writer.append({ + type: "session.started", + payload: { task: "hello" }, + }); + const second = await writer.append({ id: "fixed-id", type: "turn.done" }); + + expect(isSessionId(first.id)).toBe(true); + expect(first).toEqual({ + id: first.id, + type: "session.started", + seq: 0, + ts: 1_700_000_000_000, + payload: { task: "hello" }, + }); + expect(second.id).toBe("fixed-id"); + expect(second.seq).toBe(1); + + const lines = rawLines(dir); + expect(lines).toHaveLength(2); + for (const line of lines) { + expect(() => JSON.parse(line)).not.toThrow(); + expect(line.endsWith("\n")).toBe(false); + } + expect(lines[0]).toBe(JSON.stringify(first)); + }); + + test("preserves append order and assigns seq in order", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir); + const ids = ["a", "b", "c", "d", "e"]; + for (const id of ids) { + await writer.append({ id, type: "event" }); + } + + const records = await writer.readFrom(-1); + expect(records.map((r) => r.id)).toEqual(ids); + expect(records.map((r) => r.seq)).toEqual([0, 1, 2, 3, 4]); + + const lines = rawLines(dir).map( + (line) => JSON.parse(line) as { id: string }, + ); + expect(lines.map((l) => l.id)).toEqual(ids); + }); + + test("keeps parentId linkage resolvable across the stream", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir); + await writer.append({ id: "root", type: "session.started" }); + await writer.append({ id: "turn-1", parentId: "root", type: "turn.done" }); + await writer.append({ + id: "turn-2", + parentId: "turn-1", + type: "turn.done", + }); + + const records = await writer.readFrom(-1); + expect(records.map((r) => r.parentId)).toEqual([ + undefined, + "root", + "turn-1", + ]); + + const byId = new Map(records.map((r) => [r.id, r])); + const chain: string[] = []; + let cursor: string | undefined = "turn-2"; + while (cursor !== undefined) { + const record = byId.get(cursor); + if (record === undefined) throw new Error(`missing event ${cursor}`); + chain.push(record.id); + cursor = record.parentId; + } + expect(chain).toEqual(["turn-2", "turn-1", "root"]); + }); + + test("a reopened writer continues seq and preserves prior events", async () => { + const dir = tempDir(); + const first = createEventStreamWriter(dir); + await first.append({ id: "evt-1", type: "session.started" }); + await first.append({ id: "evt-2", type: "turn.done" }); + await first.flush(); + + const reopened = createEventStreamWriter(dir); + const third = await reopened.append({ id: "evt-3", type: "turn.done" }); + expect(third.seq).toBe(2); + + const records = await reopened.readFrom(-1); + expect(records.map((r) => r.id)).toEqual(["evt-1", "evt-2", "evt-3"]); + expect(records.map((r) => r.seq)).toEqual([0, 1, 2]); + expect(await reopened.latestCursor()).toBe(2); + }); + + test("flush awaits previously appended events", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir); + await writer.append({ id: "a", type: "event" }); + await writer.append({ id: "b", type: "event" }); + await writer.flush(); + expect(rawLines(dir)).toHaveLength(2); + }); + + test("concurrent writers on one dir cannot duplicate seqs or interleave", async () => { + const dir = tempDir(); + const writerA = createEventStreamWriter(dir); + const writerB = createEventStreamWriter(dir); + const events = await Promise.all( + Array.from({ length: 20 }, (_, i) => + (i % 2 === 0 ? writerA : writerB).append({ + id: `evt-${i}`, + type: "event", + }), + ), + ); + + const seqs = events.map((e) => e.seq); + expect(new Set(seqs).size).toBe(seqs.length); + expect(Math.max(...seqs)).toBe(seqs.length - 1); + + const records = await writerA.readFrom(-1); + expect(records).toHaveLength(20); + expect(new Set(records.map((r) => r.id)).size).toBe(20); + expect(records.map((r) => r.seq)).toEqual([...seqs].sort((a, b) => a - b)); + }); + + test("a torn tail on reopen is dropped: reads succeed and append gets a contiguous seq", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir); + await writer.append({ id: "evt-1", type: "session.started" }); + await writer.flush(); + + fs.appendFileSync( + path.join(dir, EVENT_LOG_FILE), + '{"id":"torn","type":"eve', + ); + + const reader = createEventStreamWriter(dir); + expect((await reader.readFrom(-1)).map((r) => r.id)).toEqual(["evt-1"]); + expect(await reader.latestCursor()).toBe(0); + + const reopened = createEventStreamWriter(dir); + const next = await reopened.append({ id: "evt-2", type: "turn.done" }); + expect(next.seq).toBe(1); + + const records = await reopened.readFrom(-1); + expect(records.map((r) => r.id)).toEqual(["evt-1", "evt-2"]); + expect(records.map((r) => r.seq)).toEqual([0, 1]); + }); + + test("a non-ASCII line plus a torn tail stays two parseable records with distinct seqs", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir); + await writer.append({ + id: "evt-1", + type: "session.started", + payload: { text: "é" }, + }); + await writer.flush(); + + fs.appendFileSync( + path.join(dir, EVENT_LOG_FILE), + '{"id":"torn","type":"eve', + ); + + const reopened = createEventStreamWriter(dir); + const next = await reopened.append({ id: "evt-2", type: "turn.done" }); + expect(next.seq).toBe(1); + + const lines = rawLines(dir); + expect(lines).toHaveLength(2); + const parsed = lines.map( + (line) => JSON.parse(line) as { id: string; seq: number }, + ); + expect(parsed.map((record) => record.id)).toEqual(["evt-1", "evt-2"]); + expect(parsed.map((record) => record.seq)).toEqual([0, 1]); + + const records = await reopened.readFrom(-1); + expect(records.map((r) => r.id)).toEqual(["evt-1", "evt-2"]); + expect(records.map((r) => r.seq)).toEqual([0, 1]); + const again = await reopened.append({ id: "evt-3", type: "turn.done" }); + expect(again.seq).toBe(2); + }); + + test("readFrom returns only events past the cursor", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir); + for (const id of ["a", "b", "c", "d"]) { + await writer.append({ id, type: "event" }); + } + const after = await writer.readFrom(1); + expect(after.map((r) => r.id)).toEqual(["c", "d"]); + expect(await writer.readFrom(3)).toEqual([]); + }); + + test("latestCursor reports -1 on an empty log and the last seq after appends", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir); + expect(await writer.latestCursor()).toBe(-1); + await writer.append({ id: "a", type: "event" }); + await writer.append({ id: "b", type: "event" }); + expect(await writer.latestCursor()).toBe(1); + }); + + test("projectTurns returns turn events in order", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir); + await writer.append({ id: "s", type: "session.started" }); + await writer.append({ + id: "t1", + type: "turn.user", + payload: { text: "hi" }, + }); + await writer.append({ + id: "m", + type: "model.selected", + payload: { model: "x" }, + }); + await writer.append({ + id: "t2", + type: "turn.assistant", + payload: { text: "yo" }, + }); + + const turns = (await writer.projectTurns()) as { id: string }[]; + expect(turns.map((t) => t.id)).toEqual(["t1", "t2"]); + }); + + test("projectPrompt joins text payloads in order", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir); + await writer.append({ + id: "a", + type: "turn.user", + payload: { text: "first" }, + }); + await writer.append({ + id: "b", + type: "turn.user", + payload: { text: "second" }, + }); + await writer.append({ id: "c", type: "turn.user" }); + expect(await writer.projectPrompt()).toBe("first\nsecond"); + }); + + test("backfillFromLegacy appends turns.jsonl as chained turn.legacy events", async () => { + const dir = tempDir(); + fs.writeFileSync( + path.join(dir, "turns.jsonl"), + '{"role":"user","text":"one"}\n{"role":"assistant","text":"two"}\n', + ); + const writer = createEventStreamWriter(dir); + expect(await writer.backfillFromLegacy()).toBe(2); + + const records = await writer.readFrom(-1); + expect(records).toHaveLength(2); + expect(records.every((r) => r.type === "turn.legacy")).toBe(true); + const [first, second] = records; + if (first === undefined || second === undefined) { + throw new Error("expected two backfilled events"); + } + expect(first.parentId).toBeUndefined(); + expect(second.parentId).toBe(first.id); + expect(first.payload).toMatchObject({ role: "user", text: "one" }); + }); + + test("backfillFromLegacy returns 0 when turns.jsonl is absent", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir); + expect(await writer.backfillFromLegacy()).toBe(0); + }); + + test("backfillFromLegacy is idempotent across re-runs", async () => { + const dir = tempDir(); + fs.writeFileSync( + path.join(dir, "turns.jsonl"), + '{"role":"user","text":"one"}\n{"role":"assistant","text":"two"}\n', + ); + const writer = createEventStreamWriter(dir); + expect(await writer.backfillFromLegacy()).toBe(2); + expect(await writer.backfillFromLegacy()).toBe(0); + + const records = await writer.readFrom(-1); + expect(records).toHaveLength(2); + expect(records.map((r) => r.type)).toEqual(["turn.legacy", "turn.legacy"]); + }); + + test("backfillFromLegacy skips only already-backfilled lines", async () => { + const dir = tempDir(); + fs.writeFileSync( + path.join(dir, "turns.jsonl"), + '{"role":"user","text":"one"}\n{"role":"assistant","text":"two"}\n', + ); + const writer = createEventStreamWriter(dir); + expect(await writer.backfillFromLegacy()).toBe(2); + + fs.appendFileSync( + path.join(dir, "turns.jsonl"), + '{"role":"user","text":"three"}\n', + ); + expect(await writer.backfillFromLegacy()).toBe(1); + + const records = await writer.readFrom(-1); + expect(records).toHaveLength(3); + }); + + test("backfillFromLegacy keeps identical lines and does not duplicate them on re-run", async () => { + const dir = tempDir(); + const line = '{"role":"user","text":"same"}\n'; + fs.writeFileSync(path.join(dir, "turns.jsonl"), line + line); + const writer = createEventStreamWriter(dir); + expect(await writer.backfillFromLegacy()).toBe(2); + expect(await writer.backfillFromLegacy()).toBe(0); + expect(await writer.readFrom(-1)).toHaveLength(2); + }); + + test("backfillFromLegacy includes rolled turn segments in order", async () => { + const dir = tempDir(); + fs.writeFileSync( + path.join(dir, "turns.jsonl"), + '{"role":"user","text":"one"}\n', + ); + fs.writeFileSync( + path.join(dir, "turns-0001.jsonl"), + '{"role":"user","text":"two"}\n', + ); + const writer = createEventStreamWriter(dir); + expect(await writer.backfillFromLegacy()).toBe(2); + const records = await writer.readFrom(-1); + expect( + records.map((record) => (record.payload as { text?: string }).text), + ).toEqual(["one", "two"]); + }); + + test("backfillFromLegacy stores user text where projectPrompt reads it", async () => { + const dir = tempDir(); + fs.writeFileSync( + path.join(dir, "turns.jsonl"), + '{"role":"user","text":"one"}\n', + ); + const writer = createEventStreamWriter(dir); + expect(await writer.backfillFromLegacy()).toBe(1); + expect(await writer.projectPrompt()).toBe("one"); + }); + + test("flush rejects when the append it covers rejects", async () => { + const dir = tempDir(); + const writer = createEventStreamWriter(dir); + const payload: { self?: unknown } = {}; + payload.self = payload; + const appending = writer.append({ type: "event", payload }); + const flushing = writer.flush(); + const settled = await Promise.allSettled([appending, flushing]); + expect(settled[0]?.status).toBe("rejected"); + expect(settled[1]?.status).toBe("rejected"); + await expect(appending).rejects.toThrow(TypeError); + await expect(flushing).rejects.toThrow(TypeError); + }); +}); + +describe("event stream flag gating", () => { + test("isSessionEventLogEnabled honors settings and env, with explicit off", () => { + expect(isSessionEventLogEnabled({})).toBe(false); + expect( + isSessionEventLogEnabled({ settings: { sessionEventLog: true } }), + ).toBe(true); + expect( + isSessionEventLogEnabled({ settings: { sessionEventLog: false } }), + ).toBe(false); + expect( + isSessionEventLogEnabled({ flagEnv: { [SESSION_EVENT_LOG_ENV]: "1" } }), + ).toBe(true); + expect( + isSessionEventLogEnabled({ + flagEnv: { [SESSION_EVENT_LOG_ENV]: "true" }, + }), + ).toBe(true); + for (const off of ["", "0", "false", "no", "off"]) { + expect( + isSessionEventLogEnabled({ + settings: { sessionEventLog: true }, + flagEnv: { [SESSION_EVENT_LOG_ENV]: off }, + }), + ).toBe(false); + } + }); + + test("openEventStreamWriter returns null (inert) when the flag is off", () => { + const dir = tempDir(); + expect( + openEventStreamWriter(dir, { settings: { sessionEventLog: false } }), + ).toBeNull(); + expect( + openEventStreamWriter(dir, { + flagEnv: { [SESSION_EVENT_LOG_ENV]: "off" }, + }), + ).toBeNull(); + expect(fs.existsSync(path.join(dir, EVENT_LOG_FILE))).toBe(false); + }); + + test("openEventStreamWriter returns a live writer when the flag is on", async () => { + const dir = tempDir(); + const writer = openEventStreamWriter(dir, { + settings: { sessionEventLog: true }, + }); + expect(writer).not.toBeNull(); + if (writer === null) + throw new Error("flag-enabled writer should not be null"); + await writer.append({ type: "session.started" }); + expect(await writer.readFrom(-1)).toHaveLength(1); + expect(fs.existsSync(path.join(dir, EVENT_LOG_FILE))).toBe(true); + }); +}); diff --git a/src/session/event-log.ts b/src/session/event-log.ts new file mode 100644 index 000000000..ea5c2b56d --- /dev/null +++ b/src/session/event-log.ts @@ -0,0 +1,275 @@ +import { appendFile, mkdir, readFile, truncate } from "node:fs/promises"; +import { join } from "node:path"; + +import { generateSessionId } from "./index.js"; +import { listSegmentFiles } from "./incremental-jsonl.js"; +import { withResolvedDirLock } from "./session-dir-lock.js"; + +export const EVENT_LOG_FILE = "event-log.jsonl"; + +export const SESSION_EVENT_LOG_ENV = "CORBITS_SESSION_EVENT_LOG"; + +export interface EventStreamRecord { + id: string; + parentId?: string; + type: string; + seq: number; + ts: number; + payload?: unknown; +} + +export interface AppendEventInput { + id?: string; + parentId?: string; + type: string; + payload?: unknown; +} + +export interface EventStreamWriter { + append(event: AppendEventInput): Promise; + flush(): Promise; + readFrom(cursor: number): Promise; + latestCursor(): Promise; + projectTurns(): Promise; + projectPrompt(): Promise; + backfillFromLegacy(): Promise; +} + +export interface EventStreamWriterOptions { + now?: () => number; +} + +export interface OpenEventStreamWriterOptions extends EventStreamWriterOptions { + settings?: { sessionEventLog?: boolean }; + flagEnv?: Record; +} + +export function isSessionEventLogEnabled( + sources: OpenEventStreamWriterOptions = {}, +): boolean { + const raw = (sources.flagEnv ?? process.env)[SESSION_EVENT_LOG_ENV]; + if (raw !== undefined) { + const normalized = raw.trim().toLowerCase(); + return ( + normalized !== "" && + normalized !== "0" && + normalized !== "false" && + normalized !== "no" && + normalized !== "off" + ); + } + return sources.settings?.sessionEventLog === true; +} + +function isENOENT(err: unknown): boolean { + return ( + typeof err === "object" && + err !== null && + "code" in err && + (err as { code?: unknown }).code === "ENOENT" + ); +} + +function hashLine(line: string): string { + let hash = 0x811c9dc5; + for (let i = 0; i < line.length; i++) { + hash ^= line.charCodeAt(i); + hash = Math.imul(hash, 0x01000193); + } + return (hash >>> 0).toString(16).padStart(8, "0"); +} + +// Drop an unterminated tail so a torn append does not shift seq. +function completeLines(raw: string): string[] { + if (raw.length === 0) return []; + const split = raw.split("\n"); + if (!raw.endsWith("\n")) split.pop(); + return split.filter((line) => line.length > 0); +} + +function legacyPayload( + parsed: unknown, + legacyKey: string, +): Record { + if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) { + return { ...(parsed as Record), legacyKey }; + } + return { record: parsed, legacyKey }; +} + +function parseRecordLine(line: string): EventStreamRecord { + return JSON.parse(line) as EventStreamRecord; +} + +async function readRawOrEmpty(filePath: string): Promise { + try { + return await readFile(filePath, "utf8"); + } catch (err) { + if (isENOENT(err)) return ""; + throw err; + } +} + +// Truncate the torn tail on disk so the next append cannot glue onto it. +async function dropTornTail(filePath: string, raw: string): Promise { + if (raw.length === 0 || raw.endsWith("\n")) return raw; + const lastNl = raw.lastIndexOf("\n"); + const kept = raw.slice(0, lastNl + 1); + await truncate(filePath, Buffer.byteLength(kept)); + return kept; +} + +export function createEventStreamWriter( + dir: string, + opts: EventStreamWriterOptions = {}, +): EventStreamWriter { + const now = opts.now ?? Date.now; + const filePath = join(dir, EVENT_LOG_FILE); + + let dirReady: Promise | null = null; + // flush() waits on this writer only; the dir lock orders concurrent writers. + let pending: Promise = Promise.resolve(); + + const ensureDir = async (): Promise => { + if (dirReady === null) { + dirReady = mkdir(dir, { recursive: true }).then(() => undefined); + } + await dirReady; + }; + + // Seq is the on-disk line count, so callers must already hold the dir lock. + const appendLocked = async ( + event: AppendEventInput, + ): Promise => { + await ensureDir(); + const raw = await readRawOrEmpty(filePath); + const cleaned = await dropTornTail(filePath, raw); + const seq = completeLines(cleaned).length; + const record: EventStreamRecord = { + id: event.id ?? generateSessionId(), + type: event.type, + seq, + ts: now(), + }; + if (event.parentId !== undefined) record.parentId = event.parentId; + if (event.payload !== undefined) record.payload = event.payload; + await appendFile(filePath, `${JSON.stringify(record)}\n`, "utf8"); + return record; + }; + + const append = (event: AppendEventInput): Promise => { + const run = withResolvedDirLock(dir, () => appendLocked(event)); + pending = run.then(() => undefined); + void pending.catch(() => undefined); + return run; + }; + + const flush = async (): Promise => { + const current = pending; + try { + await current; + } catch (err) { + if (pending === current) pending = Promise.resolve(); + throw err; + } + }; + + const readAll = async (): Promise => { + const raw = await readRawOrEmpty(filePath); + return completeLines(raw).map((line) => parseRecordLine(line)); + }; + + const readFrom = async (cursor: number): Promise => { + const records = await readAll(); + return records.filter((record) => record.seq > cursor); + }; + + const latestCursor = async (): Promise => { + const records = await readAll(); + const last = records[records.length - 1]; + return last === undefined ? -1 : last.seq; + }; + + const projectTurns = async (): Promise => { + const records = await readAll(); + return records + .filter((record) => record.type.startsWith("turn.")) + .map((record) => ({ ...record, ...(record.payload as object) })); + }; + + const projectPrompt = async (): Promise => { + const records = await readAll(); + const texts: string[] = []; + for (const record of records) { + const payload = record.payload as { text?: unknown } | undefined; + if (typeof payload?.text === "string" && payload.text.length > 0) { + texts.push(payload.text); + } + } + return texts.join("\n"); + }; + + const backfillFromLegacy = (): Promise => + withResolvedDirLock(dir, async () => { + const names = await listSegmentFiles(dir, "turns.jsonl"); + if (names.length === 0) return 0; + const records = await readAll(); + const existingKeys = new Set(); + for (const record of records) { + const payload = record.payload as { legacyKey?: unknown } | undefined; + if (typeof payload?.legacyKey === "string") { + existingKeys.add(payload.legacyKey); + } + } + let lastId = records[records.length - 1]?.id; + let count = 0; + const occurrences = new Map(); + for (const name of names) { + const raw = await readFile(join(dir, name), "utf8"); + for (const line of raw.split("\n")) { + if (line.length === 0) continue; + const hash = hashLine(line); + const index = occurrences.get(hash) ?? 0; + occurrences.set(hash, index + 1); + const key = `${hash}:${index}`; + if (existingKeys.has(key)) continue; + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + continue; + } + const event: AppendEventInput = { + type: "turn.legacy", + payload: legacyPayload(parsed, key), + }; + if (lastId !== undefined) event.parentId = lastId; + const appended = await appendLocked(event); + existingKeys.add(key); + lastId = appended.id; + count += 1; + } + } + return count; + }); + + return { + append, + flush, + readFrom, + latestCursor, + projectTurns, + projectPrompt, + backfillFromLegacy, + }; +} + +export function openEventStreamWriter( + dir: string, + opts: OpenEventStreamWriterOptions = {}, +): EventStreamWriter | null { + if (!isSessionEventLogEnabled(opts)) return null; + const writerOpts: EventStreamWriterOptions = {}; + if (opts.now !== undefined) writerOpts.now = opts.now; + return createEventStreamWriter(dir, writerOpts); +}