diff --git a/README.md b/README.md index 225ba1e..605af49 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,12 @@ Export uses stable capture UUIDs and bounded retries, with a database lease renewed before each outbound call. This prevents concurrent local exporters; it is not a provider-side transactional fence. Persisted desktop properties are revalidated so malformed/legacy rows cannot bypass today's privacy contract. -`posthog_state='sent'` means capture acknowledged, not erasure settled. +`posthog_state='sent'` means capture acknowledged, not erasure settled; +`'excluded'` marks rows written that way because export never sends them (desktop +diagnostics and identity observations). Event export runs from the five-minute +cron, up to ten batches of 100 per run, not after each upload, so an upload +costs one D1 round trip. Each upload logs its status, event count and stage +durations (`read_ms`, `rate_limit_ms`, `d1_ms`, `total_ms`) without identifiers. The identity-link token is service-to-service authority. Rotate it if it is exposed, and never embed it in website or desktop bundles: diff --git a/functions/_lib/analyticsPeople.ts b/functions/_lib/analyticsPeople.ts index cefa90b..07e63ab 100644 --- a/functions/_lib/analyticsPeople.ts +++ b/functions/_lib/analyticsPeople.ts @@ -42,7 +42,8 @@ export function isIdentityObservation(event: Pick, @@ -52,8 +53,12 @@ export function installationObservationStatements( const statements: D1PreparedStatement[] = []; for (const event of events) { const p = event.properties; + // The stored event proves what was observed; the token and tombstone checks prove this + // request may write it. A replay with the right event and a wrong token writes nothing. const guard = - "EXISTS (SELECT 1 FROM analytics_events WHERE event_id = ? AND distinct_id = ? AND properties_json = ? AND occurred_at = ? AND event_name = ?)"; + "EXISTS (SELECT 1 FROM analytics_events WHERE event_id = ? AND distinct_id = ? AND properties_json = ? AND occurred_at = ? AND event_name = ?)" + + " AND EXISTS (SELECT 1 FROM analytics_identities WHERE identity_id = ? AND deletion_token_hash = ?)" + + " AND NOT EXISTS (SELECT 1 FROM analytics_deleted_installations WHERE installation_id = ?)"; if (event.name === "install.account_observed") { statements.push( db @@ -75,6 +80,9 @@ export function installationObservationStatements( JSON.stringify(p), event.occurredAt, event.name, + event.distinctId, + tokenHash, + event.distinctId, ), ); } else if (event.name === "install.profile_observed") { @@ -98,6 +106,9 @@ export function installationObservationStatements( JSON.stringify(p), event.occurredAt, event.name, + event.distinctId, + tokenHash, + event.distinctId, ), ); } else if ( @@ -122,6 +133,9 @@ export function installationObservationStatements( JSON.stringify(p), event.occurredAt, event.name, + event.distinctId, + tokenHash, + event.distinctId, ), ); } diff --git a/migrations/0008_analytics_export_excluded.sql b/migrations/0008_analytics_export_excluded.sql new file mode 100644 index 0000000..87389d2 --- /dev/null +++ b/migrations/0008_analytics_export_excluded.sql @@ -0,0 +1,81 @@ +-- Rows the PostHog export never sends (desktop diagnostics and identity +-- observations) used to stay 'pending' forever, so every export re-read them. +-- They now get 'excluded'. SQLite cannot widen a CHECK constraint in place, so +-- rebuild the table. Triggers are recreated after the copy so it does not +-- recount daily totals. +CREATE TABLE analytics_events_rebuilt ( + event_id TEXT PRIMARY KEY, + event_name TEXT NOT NULL, + source TEXT NOT NULL CHECK (source IN ('website', 'desktop')), + privacy_level TEXT NOT NULL CHECK ( + privacy_level IN ('essential', 'product', 'diagnostic', 'contribution') + ), + occurred_at TEXT NOT NULL, + received_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + distinct_id TEXT NOT NULL, + properties_json TEXT NOT NULL, + posthog_state TEXT NOT NULL DEFAULT 'pending' CHECK ( + posthog_state IN ('pending', 'sent', 'excluded') + ), + posthog_attempts INTEGER NOT NULL DEFAULT 0, + posthog_last_error TEXT, + posthog_sent_at TEXT, + identity_type TEXT NOT NULL DEFAULT 'event', + canonical_id TEXT NOT NULL DEFAULT '', + session_id TEXT, + consent_level TEXT NOT NULL DEFAULT 'essential', + posthog_next_attempt_at TEXT +); + +INSERT INTO analytics_events_rebuilt ( + event_id, event_name, source, privacy_level, occurred_at, received_at, distinct_id, + properties_json, posthog_state, posthog_attempts, posthog_last_error, posthog_sent_at, + identity_type, canonical_id, session_id, consent_level, posthog_next_attempt_at +) +SELECT + event_id, event_name, source, privacy_level, occurred_at, received_at, distinct_id, + properties_json, + CASE WHEN posthog_state = 'pending' AND ( + (source = 'desktop' AND privacy_level = 'diagnostic') + OR event_name IN ('install.account_observed', 'install.profile_observed') + OR (event_name = 'install.owner_observed' AND json_valid(properties_json) + AND json_extract(properties_json, '$.ownerSource') = 'device') + ) THEN 'excluded' ELSE posthog_state END, + posthog_attempts, posthog_last_error, posthog_sent_at, + identity_type, canonical_id, session_id, consent_level, posthog_next_attempt_at +FROM analytics_events; + +DROP TABLE analytics_events; +ALTER TABLE analytics_events_rebuilt RENAME TO analytics_events; + +CREATE INDEX analytics_events_name_occurred_at ON analytics_events (event_name, occurred_at); +CREATE INDEX analytics_events_source_occurred_at ON analytics_events (source, occurred_at); +CREATE INDEX analytics_events_posthog_queue ON analytics_events (posthog_state, received_at); +CREATE INDEX analytics_events_canonical_occurred_at ON analytics_events (canonical_id, occurred_at); +CREATE INDEX analytics_events_session_occurred_at ON analytics_events (session_id, occurred_at); +CREATE INDEX analytics_events_retention ON analytics_events (received_at, event_id); + +CREATE TRIGGER analytics_no_deleted_event +BEFORE INSERT ON analytics_events +WHEN EXISTS (SELECT 1 FROM analytics_deleted_installations WHERE installation_id = NEW.distinct_id) +BEGIN + SELECT RAISE(ABORT, 'deleted-installation'); +END; + +CREATE TRIGGER analytics_daily_event_totals AFTER INSERT ON analytics_events +WHEN NEW.event_name NOT IN ('install.account_observed', 'install.profile_observed') +BEGIN + INSERT INTO analytics_daily_totals (day, metric, dimension, value, computed_at) + VALUES (substr(NEW.occurred_at, 1, 10), 'events', NEW.event_name, 1, CURRENT_TIMESTAMP) + ON CONFLICT(day, metric, dimension) DO UPDATE SET value = value + 1, computed_at = excluded.computed_at; +END; + +CREATE TRIGGER analytics_daily_active_totals AFTER INSERT ON analytics_events +WHEN NEW.source = 'desktop' AND NEW.event_name NOT IN ('install.account_observed', 'install.profile_observed') +BEGIN + INSERT OR IGNORE INTO analytics_daily_members (day, installation_id) + VALUES (substr(NEW.occurred_at, 1, 10), NEW.distinct_id); + INSERT INTO analytics_daily_totals (day, metric, dimension, value, computed_at) + SELECT substr(NEW.occurred_at, 1, 10), 'active_installations', '', 1, CURRENT_TIMESTAMP WHERE changes() = 1 + ON CONFLICT(day, metric, dimension) DO UPDATE SET value = value + 1, computed_at = excluded.computed_at; +END; diff --git a/scripts/analytics-reconciliation.mjs b/scripts/analytics-reconciliation.mjs index 7e84efa..340adbb 100644 --- a/scripts/analytics-reconciliation.mjs +++ b/scripts/analytics-reconciliation.mjs @@ -20,6 +20,7 @@ export function reconciliationQueries({ source = "desktop", from, to }) { d1: `SELECT event_name, posthog_state, COUNT(*) AS event_count FROM analytics_events WHERE source = '${source}' AND (source <> 'desktop' OR privacy_level <> 'diagnostic') + AND posthog_state <> 'excluded' AND julianday(occurred_at) >= julianday('${since}') AND julianday(occurred_at) < julianday('${until}') GROUP BY event_name, posthog_state ORDER BY event_name, posthog_state`, posthog: `SELECT event, uniqExact(properties.event_id) FROM events diff --git a/scripts/analytics-reconciliation.test.mjs b/scripts/analytics-reconciliation.test.mjs index 38f3ad1..33d4275 100644 --- a/scripts/analytics-reconciliation.test.mjs +++ b/scripts/analytics-reconciliation.test.mjs @@ -11,6 +11,8 @@ describe("analytics reconciliation", () => { } expect(query.posthog).toContain("uniqExact(properties.event_id)"); expect(query.d1).toContain("privacy_level <> 'diagnostic'"); + // Never-exported rows would otherwise fail as an unknown delivery state. + expect(query.d1).toContain("posthog_state <> 'excluded'"); expect(query.posthog).toContain("coalesce(properties.privacy_level, '') != 'diagnostic'"); expect(() => reconciliationQueries({ source: "desktop' OR 1=1", from: "2026-08-20", to: "2026-08-27" }), diff --git a/scripts/analytics-report.mjs b/scripts/analytics-report.mjs index 27b55c4..4031e44 100644 --- a/scripts/analytics-report.mjs +++ b/scripts/analytics-report.mjs @@ -140,8 +140,8 @@ export const analyticsReportQuery = ` SELECT 'posthog_delivery' AS report_section, - CASE WHEN source = 'desktop' AND privacy_level = 'diagnostic' THEN 'first-party-only' - ELSE posthog_state END AS item, + CASE WHEN posthog_state = 'excluded' OR (source = 'desktop' AND privacy_level = 'diagnostic') + THEN 'first-party-only' ELSE posthog_state END AS item, COUNT(*) AS event_count, MAX(received_at) AS latest_event FROM analytics_events diff --git a/workers/events/src/index.test.ts b/workers/events/src/index.test.ts index f0ec587..bb993c7 100644 --- a/workers/events/src/index.test.ts +++ b/workers/events/src/index.test.ts @@ -55,7 +55,12 @@ function createDatabase(existingCanonicalId?: string) { : vi.fn().mockResolvedValueOnce(null).mockResolvedValue({ authenticated: 1 }); const bind = vi.fn(() => ({ run, all, first })); const prepare = vi.fn(() => ({ bind })); - const batch = vi.fn().mockResolvedValue([]); + // Like D1, one result per statement; the ingest batch ends with its ownership check. + const batch = vi.fn(async (statements: ReadonlyArray) => + statements.map((_, index) => ({ + results: index === statements.length - 1 ? [{ authenticated: 1 }] : [], + })), + ); return { database: { prepare, batch } as unknown as D1Database, prepare, @@ -400,8 +405,10 @@ describe("event gateway routes", () => { expect(response.status).toBe(202); expect(await response.json()).toEqual({ accepted: 1 }); + // One D1 round trip: identity upsert, event insert and the ownership check. expect(database.batch).toHaveBeenCalledOnce(); - expect(database.batch.mock.calls[0]?.[0]).toHaveLength(2); + expect(database.batch.mock.calls[0]?.[0]).toHaveLength(3); + expect(database.first).not.toHaveBeenCalled(); expect(database.bind).toHaveBeenCalledWith( "installation:16ace444-e7c3-4b26-893f-98713188ae52", "installation:16ace444-e7c3-4b26-893f-98713188ae52", @@ -433,10 +440,12 @@ describe("event gateway routes", () => { "installation:16ace444-e7c3-4b26-893f-98713188ae52", "session:8e0ee7d5-2c4b-48b6-8209-08f1e536f665", "product", + "pending", "installation:16ace444-e7c3-4b26-893f-98713188ae52", expect.any(String), ); - expect(waitUntil).toHaveBeenCalledOnce(); + // Export runs from the cron, not after every ingest. + expect(waitUntil).not.toHaveBeenCalled(); }); it("rejects browser submissions from unknown origins", async () => { diff --git a/workers/events/src/index.ts b/workers/events/src/index.ts index 66c3ef7..926bdd1 100644 --- a/workers/events/src/index.ts +++ b/workers/events/src/index.ts @@ -27,6 +27,8 @@ const MAX_REQUEST_BYTES = 128 * 1024; const MAX_BATCH_SIZE = 50; const MAX_PROPERTIES_BYTES = 16 * 1024; const POSTHOG_BATCH_SIZE = 100; +// The five-minute cron is the only event export; this bounds one run's work. +const POSTHOG_MAX_BATCHES_PER_EXPORT = 10; const POSTHOG_HOST = "https://eu.i.posthog.com"; const POSTHOG_API_HOST = "https://eu.posthog.com"; const POSTHOG_DELETION_BATCH_SIZE = 1; @@ -251,11 +253,6 @@ async function persistEvents( identityEnabled = false, ): Promise { const installationId = events[0]?.distinctId; - const deleted = await database - .prepare("SELECT 1 AS deleted FROM analytics_deleted_installations WHERE installation_id = ?") - .bind(installationId) - .first(); - if (deleted) throw new InstallationAuthenticationError("Installation authentication failed"); const latestEvent = events.reduce((latest, event) => event.occurredAt > latest.occurredAt ? event : latest, ); @@ -267,14 +264,9 @@ async function persistEvents( .filter((event) => event.consentLevel === "product" || event.consentLevel === "diagnostic") .map((event) => event.occurredAt) .sort()[0] ?? null; - const existing = await database - .prepare("SELECT deletion_token_hash FROM analytics_identities WHERE identity_id = ?") - .bind(installationId) - .first<{ readonly deletion_token_hash: string | null }>(); - if (existing?.deletion_token_hash && existing.deletion_token_hash !== deletionTokenHash) { - throw new InstallationAuthenticationError("Installation authentication failed"); - } + // One D1 round trip: every write is guarded by the tombstone and token, and the + // final statement reports whether this token owns a live installation. const upsertIdentity = ` INSERT INTO analytics_identities ( identity_id, @@ -313,17 +305,18 @@ async function persistEvents( identity_type, canonical_id, session_id, - consent_level + consent_level, + posthog_state ) SELECT ?, ?, 'desktop', ?, ?, ?, ?, 'desktop_installation', COALESCE((SELECT canonical_id FROM analytics_identities WHERE identity_id = ?), ?), - ?, ? + ?, ?, ? WHERE EXISTS ( SELECT 1 FROM analytics_identities WHERE identity_id = ? AND deletion_token_hash = ? ) `; - await database.batch([ + const results = await database.batch([ database .prepare(upsertIdentity) .bind( @@ -352,6 +345,7 @@ async function persistEvents( event.distinctId, event.sessionId, event.consentLevel, + neverExported(event) ? "excluded" : "pending", event.distinctId, deletionTokenHash, ), @@ -364,19 +358,24 @@ async function persistEvents( deletionTokenHash, ) : []), + database + .prepare( + `SELECT 1 AS authenticated FROM analytics_identities + WHERE identity_id = ? AND deletion_token_hash = ? + AND NOT EXISTS (SELECT 1 FROM analytics_deleted_installations WHERE installation_id = ?)`, + ) + .bind(installationId, deletionTokenHash, installationId), ]); - - const authenticated = await database - .prepare( - "SELECT 1 AS authenticated FROM analytics_identities WHERE identity_id = ? AND deletion_token_hash = ?", - ) - .bind(installationId, deletionTokenHash) - .first<{ readonly authenticated: number }>(); - if (!authenticated) { + if ((results.at(-1)?.results.length ?? 0) === 0) { throw new InstallationAuthenticationError("Installation authentication failed"); } } +/** The PostHog export never sends these, so they never enter its queue. */ +function neverExported(event: AcceptedEvent): boolean { + return event.privacyLevel === "diagnostic" || isIdentityObservation(event); +} + async function posthogEvent(row: PendingEventRow): Promise> { let properties: Record = {}; try { @@ -430,9 +429,15 @@ async function markPosthogFailure( export async function flushPendingEvents(env: AnalyticsEnv): Promise { if (!env.POSTHOG_PROJECT_TOKEN) return 0; - return withExportLease(env.ANALYTICS_DB, (beforeRequest) => - exportPendingEvents(env, beforeRequest), - ); + return withExportLease(env.ANALYTICS_DB, async (beforeRequest) => { + let exported = 0; + for (let batch = 0; batch < POSTHOG_MAX_BATCHES_PER_EXPORT; batch += 1) { + const sent = await exportPendingEvents(env, beforeRequest); + exported += sent; + if (sent < POSTHOG_BATCH_SIZE) break; + } + return exported; + }); } async function exportPendingEvents( @@ -1136,11 +1141,7 @@ async function handleInstallationDeletion(request: Request, env: AnalyticsEnv): } } -async function handleIngestion( - request: Request, - env: AnalyticsEnv, - context: Pick, -): Promise { +async function handleIngestion(request: Request, env: AnalyticsEnv): Promise { if (env.DESKTOP_INGESTION_ENABLED !== "true") { return jsonResponse({ error: "Desktop analytics ingestion is disabled" }, 503); } @@ -1152,22 +1153,53 @@ async function handleIngestion( return jsonResponse({ error: "Content-Type must be application/json" }, 415, origin); } + // Worker clocks advance only across I/O, so these measure the body read, rate + // limiter and D1, the stages a slow acknowledgement can come from. + const started = Date.now(); + const stages: Record = {}; + let stageStart = started; + const endStage = (name: string) => { + const now = Date.now(); + stages[name] = now - stageStart; + stageStart = now; + }; + let eventCount = 0; + const respond = (response: Response, errorClass?: string) => { + console.log( + JSON.stringify({ + message: "Analytics ingestion", + status: response.status, + events: eventCount, + ...(errorClass ? { error_class: errorClass } : {}), + ...Object.fromEntries(Object.entries(stages).map(([name, ms]) => [`${name}_ms`, ms])), + total_ms: Date.now() - started, + }), + ); + return response; + }; + try { const deletionTokenHash = await sha256(requireInstallationToken(request)); const events = validateIngestionPayload( await readJsonBody(request), env.IDENTITY_ANALYTICS_ENABLED === "true", ); + endStage("read"); + eventCount = events.length; const installationId = events[0]?.distinctId; if (!installationId) throw new RequestValidationError("At least one event is required"); if (!env.ANALYTICS_INGESTION_RATE_LIMITER) { - return jsonResponse({ error: "Analytics rate limiting is unavailable" }, 503, origin); + return respond( + jsonResponse({ error: "Analytics rate limiting is unavailable" }, 503, origin), + "rate-limiter-unavailable", + ); } const rateLimit = await env.ANALYTICS_INGESTION_RATE_LIMITER.limit({ key: installationId }); + endStage("rate_limit"); if (!rateLimit.success) { const response = jsonResponse({ error: "Too many analytics requests" }, 429, origin); response.headers.set("Retry-After", "60"); - return response; + return respond(response, "rate-limited"); } await persistEvents( env.ANALYTICS_DB, @@ -1176,31 +1208,21 @@ async function handleIngestion( request, env.IDENTITY_ANALYTICS_ENABLED === "true", ); - context.waitUntil( - flushPendingEvents(env).catch((error: unknown) => { - console.error( - JSON.stringify({ - message: "PostHog forwarding failed", - error_class: error instanceof TransportFailure ? error.kind : "internal", - }), - ); - }), - ); - return jsonResponse({ accepted: events.length }, 202, origin); + endStage("d1"); + return respond(jsonResponse({ accepted: events.length }, 202, origin)); } catch (error) { + endStage("failed"); if (error instanceof InstallationAuthenticationError) { - return jsonResponse({ error: error.message }, 403, origin); + return respond(jsonResponse({ error: error.message }, 403, origin), "authentication"); } if (error instanceof RequestValidationError) { - return jsonResponse({ error: error.message }, 400, origin); + return respond(jsonResponse({ error: error.message }, 400, origin), "validation"); } + const errorClass = error instanceof TransportFailure ? error.kind : "internal"; console.error( - JSON.stringify({ - message: "Analytics ingestion failed", - error_class: error instanceof TransportFailure ? error.kind : "internal", - }), + JSON.stringify({ message: "Analytics ingestion failed", error_class: errorClass }), ); - return jsonResponse({ error: "Analytics ingestion failed" }, 500, origin); + return respond(jsonResponse({ error: "Analytics ingestion failed" }, 500, origin), errorClass); } } @@ -1278,7 +1300,7 @@ const worker = { return response; } if (request.method === "POST" && url.pathname === "/v1/events") { - return handleIngestion(request, env, context); + return handleIngestion(request, env); } if (request.method === "POST" && url.pathname === "/v1/identity/link") { return handleIdentityLink(request, env, context); diff --git a/workers/events/src/people.test.ts b/workers/events/src/people.test.ts index 3981cc0..c8238dd 100644 --- a/workers/events/src/people.test.ts +++ b/workers/events/src/people.test.ts @@ -164,6 +164,20 @@ it("rejects wrong tokens and ignores event replays without inflating totals or c ).toBe(1); }); +it("writes nothing for an exact observation replay sent with a wrong token", async () => { + const f = fixture(); + const id = crypto.randomUUID(); + const at = new Date().toISOString(); + const observation = { provider: "codex", email: "one@scient.test" }; + expect((await f.send("install.account_observed", observation, id, token, at)).status).toBe(202); + const changes = () => f.sqlite.prepare("SELECT total_changes() AS n").get()!.n; + const before = changes(); + expect( + (await f.send("install.account_observed", observation, id, "b".repeat(64), at)).status, + ).toBe(403); + expect(changes()).toBe(before); +}); + it("erases installation observations, preserves aggregates and prevents resurrection", async () => { const f = fixture(); await f.send("install.account_observed", { provider: "codex", email: "one@scient.test" }); diff --git a/workers/events/src/readiness.test.ts b/workers/events/src/readiness.test.ts index 975e81d..7f1280f 100644 --- a/workers/events/src/readiness.test.ts +++ b/workers/events/src/readiness.test.ts @@ -11,8 +11,8 @@ import { withExportLease } from "./exportLease"; const installation = "installation:10000000-0000-4000-8000-000000000001"; const token = "a".repeat(64); const databases: ReturnType[] = []; -function fixture(beforeReadiness?: Parameters[0]) { - const store = testDatabase(beforeReadiness); +function fixture(...hooks: Parameters) { + const store = testDatabase(...hooks); databases.push(store); const tasks: Promise[] = []; const env = { @@ -153,6 +153,113 @@ describe("inactive gateway readiness with real SQL", () => { expect(await pruneExpiredAnalyticsEvents(f.database)).toBe(1); }); + it("writes never-exported rows as excluded and leaves export to the cron", async () => { + const f = fixture(); + const session = f.event().events[0]!; + const diagnostic = { + ...session, + id: crypto.randomUUID(), + name: "app.diagnostics", + privacy_level: "diagnostic", + consent_level: "diagnostic", + properties: { + appVersion: "0.6.8", + buildChannel: "stable", + queuedCountBucket: "0", + droppedCountBucket: "0", + retryCountBucket: "0", + deliveryClass: "delivered", + }, + }; + const response = await f.request("/v1/events", { + ...f.event(), + events: [session, { ...diagnostic, consent_level: "diagnostic" }], + }); + expect(response.status).toBe(202); + expect(f.tasks).toHaveLength(0); + expect( + f.sqlite + .prepare("SELECT event_name, posthog_state FROM analytics_events ORDER BY event_name") + .all(), + ).toEqual([ + { event_name: "app.diagnostics", posthog_state: "excluded" }, + { event_name: "app.session.started", posthog_state: "pending" }, + ]); + }); + + it("drains several PostHog batches in one cron export", async () => { + const f = fixture(); + for (let batch = 0; batch < 5; batch += 1) { + const events = Array.from({ length: 50 }, () => ({ + ...f.event().events[0]!, + id: crypto.randomUUID(), + })); + expect((await f.request("/v1/events", { ...f.event(), events })).status).toBe(202); + } + const fetcher = vi.fn().mockResolvedValue(new Response(null, { status: 200 })); + vi.stubGlobal("fetch", fetcher); + expect( + await flushPendingEvents({ + ...f.env, + POSTHOG_PROJECT_TOKEN: "synthetic", + DESKTOP_POSTHOG_EXPORT_ENABLED: "true", + }), + ).toBe(250); + expect(fetcher).toHaveBeenCalledTimes(3); + }); + + it("backfills excluded rows without recounting totals and keeps the table's triggers", async () => { + const insert = `INSERT INTO analytics_events (event_id, event_name, source, privacy_level, + occurred_at, distinct_id, properties_json, posthog_state) VALUES (?, ?, 'desktop', ?, ?, ?, ?, ?)`; + const day = "2026-11-14T10:00:00.000Z"; + const f = fixture(undefined, { + "0008_analytics_export_excluded.sql": (sqlite) => { + const rows = [ + ["diagnostic", "app.diagnostics", "diagnostic", "{}", "pending"], + ["account", "install.account_observed", "product", "{}", "pending"], + ["device", "install.owner_observed", "product", '{"ownerSource":"device"}', "pending"], + ["owner", "install.owner_observed", "product", '{"ownerSource":"account"}', "pending"], + ["product", "provider.turn.sent", "product", "{}", "pending"], + ["sent", "app.diagnostics", "diagnostic", "{}", "sent"], + ] as const; + for (const [id, name, privacy, properties, state] of rows) { + sqlite.prepare(insert).run(id, name, privacy, day, installation, properties, state); + } + }, + }); + expect( + f.sqlite + .prepare("SELECT event_id, posthog_state FROM analytics_events ORDER BY event_id") + .all(), + ).toEqual([ + { event_id: "account", posthog_state: "excluded" }, + { event_id: "device", posthog_state: "excluded" }, + { event_id: "diagnostic", posthog_state: "excluded" }, + { event_id: "owner", posthog_state: "pending" }, + { event_id: "product", posthog_state: "pending" }, + { event_id: "sent", posthog_state: "sent" }, + ]); + const eventTotal = () => + f.sqlite + .prepare( + "SELECT sum(value) AS n FROM analytics_daily_totals WHERE metric = 'events' AND day = '2026-11-14'", + ) + .get()!.n; + expect(eventTotal()).toBe(5); + f.sqlite + .prepare(insert) + .run("later", "provider.turn.sent", "product", day, installation, "{}", "pending"); + expect(eventTotal()).toBe(6); + f.sqlite.exec(`INSERT INTO analytics_deleted_installations + (installation_id, deletion_token_hash, request_id, requested_at) + VALUES ('${installation}', 'hash', 'request', '${day}')`); + expect(() => + f.sqlite + .prepare(insert) + .run("deleted", "provider.turn.sent", "product", day, installation, "{}", "pending"), + ).toThrow("deleted-installation"); + }); + it("does not reset diagnostic retention when old offline events arrive or await export", async () => { const f = fixture(); const old = new Date(Date.now() - 31 * 86400000).toISOString(); diff --git a/workers/events/src/sqlite.testSupport.ts b/workers/events/src/sqlite.testSupport.ts index fcc3019..8b47b26 100644 --- a/workers/events/src/sqlite.testSupport.ts +++ b/workers/events/src/sqlite.testSupport.ts @@ -3,13 +3,17 @@ import { readFileSync, readdirSync } from "node:fs"; import { DatabaseSync, type SQLInputValue } from "node:sqlite"; /** Exercises real migration/transaction SQL; not a substitute for D1 deployment proof. */ -export function testDatabase(beforeReadiness?: (sqlite: DatabaseSync) => void) { +export function testDatabase( + beforeReadiness?: (sqlite: DatabaseSync) => void, + beforeMigration: Readonly void>> = {}, +) { const sqlite = new DatabaseSync(":memory:"); const migrations = new URL("../../../migrations/", import.meta.url); for (const file of readdirSync(migrations) .filter((name) => name.endsWith(".sql")) .sort()) { if (file === "0006_analytics_readiness.sql") beforeReadiness?.(sqlite); + beforeMigration[file]?.(sqlite); sqlite.exec(readFileSync(new URL(file, migrations), "utf8")); } const prepare = (sql: string, bindings: SQLInputValue[] = []) => {