Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,12 @@ Export uses stable capture UUIDs and bounded retries, with a database lease
renewed before each outbound call. This prevents concurrent local exporters;
it is not a provider-side transactional fence. Persisted desktop properties are
revalidated so malformed/legacy rows cannot bypass today's privacy contract.
`posthog_state='sent'` means capture acknowledged, not erasure settled.
`posthog_state='sent'` means capture acknowledged, not erasure settled;
`'excluded'` marks rows written that way because export never sends them (desktop
diagnostics and identity observations). Event export runs from the five-minute
cron, up to ten batches of 100 per run, not after each upload, so an upload
costs one D1 round trip. Each upload logs its status, event count and stage
durations (`read_ms`, `rate_limit_ms`, `d1_ms`, `total_ms`) without identifiers.

The identity-link token is service-to-service authority. Rotate it if it is exposed, and never embed it in website or desktop bundles:

Expand Down
18 changes: 16 additions & 2 deletions functions/_lib/analyticsPeople.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ export function isIdentityObservation(event: Pick<IdentityObservation, "name" |
);
}

// Called inside the event transaction, after token ownership and tombstones are checked.
// Runs inside the event batch. Every write requires the installation's token and no tombstone;
// observation writes also require their stored event.
export function installationObservationStatements(
db: D1Database,
events: ReadonlyArray<IdentityObservation>,
Expand All @@ -52,8 +53,12 @@ export function installationObservationStatements(
const statements: D1PreparedStatement[] = [];
for (const event of events) {
const p = event.properties;
// The stored event proves what was observed; the token and tombstone checks prove this
// request may write it. A replay with the right event and a wrong token writes nothing.
const guard =
"EXISTS (SELECT 1 FROM analytics_events WHERE event_id = ? AND distinct_id = ? AND properties_json = ? AND occurred_at = ? AND event_name = ?)";
"EXISTS (SELECT 1 FROM analytics_events WHERE event_id = ? AND distinct_id = ? AND properties_json = ? AND occurred_at = ? AND event_name = ?)" +
" AND EXISTS (SELECT 1 FROM analytics_identities WHERE identity_id = ? AND deletion_token_hash = ?)" +
" AND NOT EXISTS (SELECT 1 FROM analytics_deleted_installations WHERE installation_id = ?)";
if (event.name === "install.account_observed") {
statements.push(
db
Expand All @@ -75,6 +80,9 @@ export function installationObservationStatements(
JSON.stringify(p),
event.occurredAt,
event.name,
event.distinctId,
tokenHash,
event.distinctId,
),
);
} else if (event.name === "install.profile_observed") {
Expand All @@ -98,6 +106,9 @@ export function installationObservationStatements(
JSON.stringify(p),
event.occurredAt,
event.name,
event.distinctId,
tokenHash,
event.distinctId,
),
);
} else if (
Expand All @@ -122,6 +133,9 @@ export function installationObservationStatements(
JSON.stringify(p),
event.occurredAt,
event.name,
event.distinctId,
tokenHash,
event.distinctId,
),
);
}
Expand Down
81 changes: 81 additions & 0 deletions migrations/0008_analytics_export_excluded.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
-- Rows the PostHog export never sends (desktop diagnostics and identity
-- observations) used to stay 'pending' forever, so every export re-read them.
-- They now get 'excluded'. SQLite cannot widen a CHECK constraint in place, so
-- rebuild the table. Triggers are recreated after the copy so it does not
-- recount daily totals.
CREATE TABLE analytics_events_rebuilt (
event_id TEXT PRIMARY KEY,
event_name TEXT NOT NULL,
source TEXT NOT NULL CHECK (source IN ('website', 'desktop')),
privacy_level TEXT NOT NULL CHECK (
privacy_level IN ('essential', 'product', 'diagnostic', 'contribution')
),
occurred_at TEXT NOT NULL,
received_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
distinct_id TEXT NOT NULL,
properties_json TEXT NOT NULL,
posthog_state TEXT NOT NULL DEFAULT 'pending' CHECK (
posthog_state IN ('pending', 'sent', 'excluded')
),
posthog_attempts INTEGER NOT NULL DEFAULT 0,
posthog_last_error TEXT,
posthog_sent_at TEXT,
identity_type TEXT NOT NULL DEFAULT 'event',
canonical_id TEXT NOT NULL DEFAULT '',
session_id TEXT,
consent_level TEXT NOT NULL DEFAULT 'essential',
posthog_next_attempt_at TEXT
);

INSERT INTO analytics_events_rebuilt (
event_id, event_name, source, privacy_level, occurred_at, received_at, distinct_id,
properties_json, posthog_state, posthog_attempts, posthog_last_error, posthog_sent_at,
identity_type, canonical_id, session_id, consent_level, posthog_next_attempt_at
)
SELECT
event_id, event_name, source, privacy_level, occurred_at, received_at, distinct_id,
properties_json,
CASE WHEN posthog_state = 'pending' AND (
(source = 'desktop' AND privacy_level = 'diagnostic')
OR event_name IN ('install.account_observed', 'install.profile_observed')
OR (event_name = 'install.owner_observed' AND json_valid(properties_json)
AND json_extract(properties_json, '$.ownerSource') = 'device')
) THEN 'excluded' ELSE posthog_state END,
posthog_attempts, posthog_last_error, posthog_sent_at,
identity_type, canonical_id, session_id, consent_level, posthog_next_attempt_at
FROM analytics_events;

DROP TABLE analytics_events;
ALTER TABLE analytics_events_rebuilt RENAME TO analytics_events;

CREATE INDEX analytics_events_name_occurred_at ON analytics_events (event_name, occurred_at);
CREATE INDEX analytics_events_source_occurred_at ON analytics_events (source, occurred_at);
CREATE INDEX analytics_events_posthog_queue ON analytics_events (posthog_state, received_at);
CREATE INDEX analytics_events_canonical_occurred_at ON analytics_events (canonical_id, occurred_at);
CREATE INDEX analytics_events_session_occurred_at ON analytics_events (session_id, occurred_at);
CREATE INDEX analytics_events_retention ON analytics_events (received_at, event_id);

CREATE TRIGGER analytics_no_deleted_event
BEFORE INSERT ON analytics_events
WHEN EXISTS (SELECT 1 FROM analytics_deleted_installations WHERE installation_id = NEW.distinct_id)
BEGIN
SELECT RAISE(ABORT, 'deleted-installation');
END;

CREATE TRIGGER analytics_daily_event_totals AFTER INSERT ON analytics_events
WHEN NEW.event_name NOT IN ('install.account_observed', 'install.profile_observed')
BEGIN
INSERT INTO analytics_daily_totals (day, metric, dimension, value, computed_at)
VALUES (substr(NEW.occurred_at, 1, 10), 'events', NEW.event_name, 1, CURRENT_TIMESTAMP)
ON CONFLICT(day, metric, dimension) DO UPDATE SET value = value + 1, computed_at = excluded.computed_at;
END;

CREATE TRIGGER analytics_daily_active_totals AFTER INSERT ON analytics_events
WHEN NEW.source = 'desktop' AND NEW.event_name NOT IN ('install.account_observed', 'install.profile_observed')
BEGIN
INSERT OR IGNORE INTO analytics_daily_members (day, installation_id)
VALUES (substr(NEW.occurred_at, 1, 10), NEW.distinct_id);
INSERT INTO analytics_daily_totals (day, metric, dimension, value, computed_at)
SELECT substr(NEW.occurred_at, 1, 10), 'active_installations', '', 1, CURRENT_TIMESTAMP WHERE changes() = 1
ON CONFLICT(day, metric, dimension) DO UPDATE SET value = value + 1, computed_at = excluded.computed_at;
END;
1 change: 1 addition & 0 deletions scripts/analytics-reconciliation.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ export function reconciliationQueries({ source = "desktop", from, to }) {
d1: `SELECT event_name, posthog_state, COUNT(*) AS event_count
FROM analytics_events WHERE source = '${source}'
AND (source <> 'desktop' OR privacy_level <> 'diagnostic')
AND posthog_state <> 'excluded'
AND julianday(occurred_at) >= julianday('${since}') AND julianday(occurred_at) < julianday('${until}')
GROUP BY event_name, posthog_state ORDER BY event_name, posthog_state`,
posthog: `SELECT event, uniqExact(properties.event_id) FROM events
Expand Down
2 changes: 2 additions & 0 deletions scripts/analytics-reconciliation.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ describe("analytics reconciliation", () => {
}
expect(query.posthog).toContain("uniqExact(properties.event_id)");
expect(query.d1).toContain("privacy_level <> 'diagnostic'");
// Never-exported rows would otherwise fail as an unknown delivery state.
expect(query.d1).toContain("posthog_state <> 'excluded'");
expect(query.posthog).toContain("coalesce(properties.privacy_level, '') != 'diagnostic'");
expect(() =>
reconciliationQueries({ source: "desktop' OR 1=1", from: "2026-08-20", to: "2026-08-27" }),
Expand Down
4 changes: 2 additions & 2 deletions scripts/analytics-report.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,8 @@ export const analyticsReportQuery = `

SELECT
'posthog_delivery' AS report_section,
CASE WHEN source = 'desktop' AND privacy_level = 'diagnostic' THEN 'first-party-only'
ELSE posthog_state END AS item,
CASE WHEN posthog_state = 'excluded' OR (source = 'desktop' AND privacy_level = 'diagnostic')
THEN 'first-party-only' ELSE posthog_state END AS item,
COUNT(*) AS event_count,
MAX(received_at) AS latest_event
FROM analytics_events
Expand Down
15 changes: 12 additions & 3 deletions workers/events/src/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,12 @@ function createDatabase(existingCanonicalId?: string) {
: vi.fn().mockResolvedValueOnce(null).mockResolvedValue({ authenticated: 1 });
const bind = vi.fn(() => ({ run, all, first }));
const prepare = vi.fn(() => ({ bind }));
const batch = vi.fn().mockResolvedValue([]);
// Like D1, one result per statement; the ingest batch ends with its ownership check.
const batch = vi.fn(async (statements: ReadonlyArray<unknown>) =>
statements.map((_, index) => ({
results: index === statements.length - 1 ? [{ authenticated: 1 }] : [],
})),
);
return {
database: { prepare, batch } as unknown as D1Database,
prepare,
Expand Down Expand Up @@ -400,8 +405,10 @@ describe("event gateway routes", () => {

expect(response.status).toBe(202);
expect(await response.json()).toEqual({ accepted: 1 });
// One D1 round trip: identity upsert, event insert and the ownership check.
expect(database.batch).toHaveBeenCalledOnce();
expect(database.batch.mock.calls[0]?.[0]).toHaveLength(2);
expect(database.batch.mock.calls[0]?.[0]).toHaveLength(3);
expect(database.first).not.toHaveBeenCalled();
expect(database.bind).toHaveBeenCalledWith(
"installation:16ace444-e7c3-4b26-893f-98713188ae52",
"installation:16ace444-e7c3-4b26-893f-98713188ae52",
Expand Down Expand Up @@ -433,10 +440,12 @@ describe("event gateway routes", () => {
"installation:16ace444-e7c3-4b26-893f-98713188ae52",
"session:8e0ee7d5-2c4b-48b6-8209-08f1e536f665",
"product",
"pending",
"installation:16ace444-e7c3-4b26-893f-98713188ae52",
expect.any(String),
);
expect(waitUntil).toHaveBeenCalledOnce();
// Export runs from the cron, not after every ingest.
expect(waitUntil).not.toHaveBeenCalled();
});

it("rejects browser submissions from unknown origins", async () => {
Expand Down
Loading