diff --git a/.github/workflows/digest.yml b/.github/workflows/digest.yml new file mode 100644 index 0000000..9ce91df --- /dev/null +++ b/.github/workflows/digest.yml @@ -0,0 +1,61 @@ +name: Observability Digest + +on: + schedule: + - cron: "0 16 * * 1" # Mondays 16:00 UTC + workflow_dispatch: + inputs: + days: + description: "Days to review (default: 7)" + required: false + type: number + post: + description: "Post to Slack (off prints the digest to the log only)" + required: false + type: boolean + default: true + +permissions: + contents: read + actions: write # keepalive below + +jobs: + digest: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout sourcebot + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: "npm" + + - name: Install dependencies + run: npm ci + + - name: Run digest + run: npx tsx src/digest.ts + env: + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + SC_DEV_BOT_APP_ID: ${{ vars.SC_DEV_BOT_APP_ID }} + SC_DEV_BOT_PRIVATE_KEY: ${{ secrets.SC_DEV_BOT_PRIVATE_KEY }} + SC_DEV_BOT_INSTALLATION_ID: ${{ vars.SC_DEV_BOT_INSTALLATION_ID }} + SLACK_WEBHOOK_URL: ${{ (github.event_name == 'schedule' || inputs.post) && secrets.SLACK_WEBHOOK_URL || '' }} + DIGEST_DAYS: ${{ inputs.days }} + + # GitHub disables scheduled workflows after 60 days without repository + # activity (that stopped the monitor on 2026-10-10). Re-enabling them from + # a scheduled run keeps them on. + - name: Keep scheduled workflows enabled + if: always() + run: | + gh workflow enable monitor.yml -R "$GITHUB_REPOSITORY" + gh workflow enable digest.yml -R "$GITHUB_REPOSITORY" + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/monitor.yml b/.github/workflows/monitor.yml index 4d041d5..8b6a828 100644 --- a/.github/workflows/monitor.yml +++ b/.github/workflows/monitor.yml @@ -10,6 +10,10 @@ on: required: false type: number +permissions: + contents: read + actions: write # keepalive below + jobs: monitor: runs-on: ubuntu-latest @@ -41,3 +45,14 @@ jobs: SC_DEV_BOT_PRIVATE_KEY: ${{ secrets.SC_DEV_BOT_PRIVATE_KEY }} SC_DEV_BOT_INSTALLATION_ID: ${{ vars.SC_DEV_BOT_INSTALLATION_ID }} WINDOW_HOURS: ${{ inputs.window_hours }} + + # GitHub disables scheduled workflows after 60 days without repository + # activity (that stopped this monitor on 2026-10-10). Re-enabling them from + # a scheduled run keeps them on. + - name: Keep scheduled workflows enabled + if: always() + run: | + gh workflow enable monitor.yml -R "$GITHUB_REPOSITORY" + gh workflow enable digest.yml -R "$GITHUB_REPOSITORY" + env: + GH_TOKEN: ${{ github.token }} diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index aaaa7b9..a64b45b 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -36,6 +36,7 @@ At https://dash.cloudflare.com/profile/api-tokens, create a Custom Token with: - Account → D1: Edit - Account → Workers Observability: Read - Account → Account Analytics: Read +- Account → Notifications: Read (the digest reads alert policies and history) Scope to your specific account. @@ -67,6 +68,9 @@ At https://console.anthropic.com/settings/keys, create a key named `sourcebot`. | Variable | `VERCEL_TEAM_ID` | | Secret | `CLOUDFLARE_API_TOKEN` | | Secret | `VERCEL_API_TOKEN` | +| Secret | `SLACK_WEBHOOK_URL` | + +`SLACK_WEBHOOK_URL` is a Slack incoming webhook for the channel the weekly digest posts to (#devops). Without it the digest prints to the workflow log instead of posting. ### 7. Initialize the D1 Schema @@ -98,12 +102,19 @@ repos: - name: source-cooperative/data.source.coop log_source: cloudflare_workers - cloudflare_script_name: data-source-coop - auto_fix: true + cloudflare_script_name: source-data-proxy + cloudflare_issues_since: "2026-11-19T00:00:00Z" # older Workers Issues groups are not filed + auto_fix: false schedule: "0 */6 * * *" comment_cadence_days: 7 anthropic_model: claude-sonnet-4-6 + +digest: + repo: source-cooperative/data.source.coop + script_name: source-data-proxy + analytics_dataset: source_data_proxy_production + days: 7 ``` When `auto_fix: true`, the monitor self-assigns new issues to Scooter, triggering the fix workflow automatically. @@ -115,6 +126,7 @@ npm install npm test # Run test suite (vitest) npm run lint # Type check (tsc --noEmit) npm run monitor # Run the monitor locally (requires env vars) +npm run digest # Run the digest locally; prints instead of posting unless SLACK_WEBHOOK_URL is set ``` ## Project Structure @@ -123,6 +135,7 @@ npm run monitor # Run the monitor locally (requires env vars) sourcebot/ ├── .github/workflows/ │ ├── monitor.yml Cron monitor +│ ├── digest.yml Weekly observability digest │ ├── fix.yml Reusable fix workflow │ └── ci.yml Test and type-check on PRs ├── caller-workflows/ @@ -131,6 +144,7 @@ sourcebot/ │ └── init-d1.sh One-time schema setup ├── src/ │ ├── index.ts Main orchestrator + CLI entry +│ ├── digest.ts Weekly observability digest + CLI entry │ ├── config.ts Load config.yaml │ ├── d1.ts D1 REST API client │ ├── github.ts GitHub App auth + issue management @@ -138,7 +152,7 @@ sourcebot/ │ ├── classifier.ts Anthropic API for error grouping │ └── sources/ │ ├── vercel.ts Vercel runtime logs -│ └── cloudflare.ts Workers Observability +│ └── cloudflare.ts Workers Observability Issues ├── config.yaml Which repos to monitor └── schema.sql D1 schema ``` diff --git a/README.md b/README.md index 9d20803..c54618b 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,20 @@ The GitHub App is named **Scooter** (Source Cooperative Operator for Observation **Monitor workflow** (runs every 6 hours): - Fetches runtime errors from Vercel for `source.coop` -- Fetches error-level logs from Cloudflare Workers Observability for `data.source.coop` +- Fetches Cloudflare's Workers Observability Issues (its own grouping of exceptions, 5xx responses and ERROR lines) for `data.source.coop`, skipping vulnerability-scan detections - Computes a fingerprint for each error (normalized message + stack location + status code) - Groups new errors via the Anthropic API and creates GitHub issues in the appropriate target repo - Reopens closed issues when the same error recurs on a newer release version - Comments weekly on open issues with updated occurrence counts +**Digest workflow** (Mondays 16:00 UTC): + +- Gathers the past week of the data proxy's observability: alert notifications and monitor definitions, Workers Issues, Analytics Engine traffic (daily requests and 5xx, read p99 and its worst 15-minute windows, top 5xx and busiest products), Workers Logs WARN/ERROR volume and top sources, and `data.source.coop`'s open issues +- Has Claude write a short review (which alerts were real, what drove errors and latency, which noise sources are tracked by an open issue and which are not, suggested actions) and posts it to Slack +- Only suggests: it files no issues and changes no monitors + +Both scheduled workflows re-enable themselves on each run, so GitHub's 60-day inactivity rule does not switch them off. + **Fix workflow** (triggered by issue assignment): - Runs when a human assigns an issue to `scooter[bot]` @@ -29,6 +37,7 @@ State lives in Cloudflare D1, accessed via its REST API. ``` GitHub Actions (this repo) ├── monitor.yml Cron: fetch logs → classify → create/update issues +├── digest.yml Weekly: gather proxy observability → summarize → Slack └── fix.yml Reusable: called from target repos on issue assignment Target repos (source.coop, data.source.coop) diff --git a/config.yaml b/config.yaml index 015b9fa..2c2b2b5 100644 --- a/config.yaml +++ b/config.yaml @@ -7,11 +7,23 @@ repos: - name: source-cooperative/data.source.coop log_source: cloudflare_workers - cloudflare_script_name: data-source-coop + cloudflare_script_name: source-data-proxy + # Workers Issues groups that already existed when sourcebot started reading + # Issues are tracked by hand (e.g. data.source.coop#206, #281); only file new ones. + cloudflare_issues_since: "2026-11-19T00:00:00Z" dashboard_url: https://dash.cloudflare.com/1335bb5b87a1c2e1de2ca03408c23b71/workers/services/view/source-data-proxy/production/observability/invocations - auto_fix: true + # Most proxy error groups are infrastructure (Cloudflare 52x, backend STS), + # not code a bot can fix; a human assigns Scooter when one is. + auto_fix: false schedule: "0 */6 * * *" window_hours: 6 comment_cadence_days: 7 anthropic_model: claude-sonnet-4-6 + +# Weekly digest of the data proxy's observability, posted to Slack (src/digest.ts). +digest: + repo: source-cooperative/data.source.coop + script_name: source-data-proxy + analytics_dataset: source_data_proxy_production + days: 7 diff --git a/package.json b/package.json index fdd4218..df1a262 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "type": "module", "scripts": { "monitor": "tsx src/index.ts", + "digest": "tsx src/digest.ts", "test": "vitest run", "test:watch": "vitest", "lint": "tsc --noEmit" diff --git a/src/classifier.ts b/src/classifier.ts index 4bde6b4..327693a 100644 --- a/src/classifier.ts +++ b/src/classifier.ts @@ -46,7 +46,7 @@ export class ErrorClassifier { HTTP Status: ${e.httpStatus ?? "N/A"} Repo: ${e.source} Release: ${e.releaseVersion} - Occurrences (last window): ${e.count}` + Occurrences: ${e.count}` ) .join("\n\n"); diff --git a/src/config.ts b/src/config.ts index b6d0dfa..3731857 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,12 +7,16 @@ export interface RepoConfig { log_source: "vercel" | "cloudflare_workers"; vercel_project_id?: string; cloudflare_script_name?: string; + /** Workers Issues groups first seen before this ISO date are not filed. */ + cloudflare_issues_since?: string; dashboard_url?: string; auto_fix: boolean; } export interface Config { repos: RepoConfig[]; + /** Weekly observability digest posted to Slack (src/digest.ts). */ + digest?: { repo: string; script_name: string; analytics_dataset: string; days: number }; schedule: string; comment_cadence_days: number; anthropic_model: string; diff --git a/src/digest.test.ts b/src/digest.test.ts new file mode 100644 index 0000000..df92134 --- /dev/null +++ b/src/digest.test.ts @@ -0,0 +1,99 @@ +import { describe, it, expect, vi } from "vitest"; +import { runDigest, normalizeLogMessage, type DigestDeps } from "./digest.js"; + +const cfg = { + repo: "source-cooperative/data.source.coop", + script_name: "source-data-proxy", + analytics_dataset: "source_data_proxy_production", + days: 7, +}; +const NOW = Date.parse("2026-11-19T12:00:00Z"); + +function deps(over: Partial = {}): DigestDeps { + return { + accountId: "acct", + now: () => NOW, + cf: vi.fn(async (path: string, init?: { body?: string }) => { + if (path.startsWith("/alerting/v3/policies")) + return { result: [{ name: "data-proxy: M7 Latency", enabled: true, description: "p99 > 5 s" }, { name: "other", enabled: true }] }; + if (path.startsWith("/alerting/v3/history")) + return { + result: [ + { name: "data-proxy: M7 Latency", sent: "2026-11-17T19:58:00Z", mechanism_type: "webhook", alert_body: "Notification Policy: data-proxy: M7 Latency\nTriggered Result: 9414" }, + { name: "data-proxy: M7 Latency", sent: "2026-11-17T19:58:00Z", mechanism_type: "email", alert_body: "{}" }, + { name: "data-proxy: M7 Latency", sent: "2026-11-17T20:13:00Z", mechanism_type: "webhook", alert_body: "Alert State: Resolved\nTriggered Result: 1571" }, + ], + }; + if (path.startsWith("/workers/observability/issues")) + return { result: [{ title: "HttpServerError · HTTP 520", type: "exception", count: 9754, firstObserved: NOW - 9e8, lastObserved: NOW - 1000 }, { title: "old", count: 1, firstObserved: 0, lastObserved: 0 }] }; + if (path === "/analytics/sql") { + if (init?.body?.includes("substring(message")) + return { + data: [ + { level: "warn", msg: "[WARN] request failed { request_id=aaa, duration_ms=37.0, path=/ftw/benchmark-data/a/b.webp }", n: 10 }, + { level: "warn", msg: "[WARN] request failed { request_id=bbb, duration_ms=12.0, path=/ftw/benchmark-data/c.webp }", n: 5 }, + ], + }; + return { data: [{ day: "2026-11-18", level: "warn", n: 40000 }] }; + } + if (path === "/analytics_engine/sql") { + if (init?.body?.includes("AS bucket")) + return { data: [{ bucket: "a", read_p99_ms: 9000, requests: "10" }, { bucket: "b", read_p99_ms: 3600, requests: "400000" }] }; + return { data: [{ day: "2026-11-18", requests: "30000000" }] }; + } + throw new Error(`unexpected ${path}`); + }), + listOpenIssues: vi.fn(async () => [{ number: 206, title: "Streamed PUTs intermittently 520" }]), + summarize: vi.fn(async () => "*Headline*: fine."), + post: vi.fn(async () => undefined), + ...over, + }; +} + +function facts(d: DigestDeps): any { + const prompt = (d.summarize as any).mock.calls[0][1] as string; + return JSON.parse(prompt.slice(prompt.indexOf("{"))); +} + +describe("runDigest", () => { + it("gathers the week's data, has it summarized, and posts the summary", async () => { + const d = deps(); + const text = await runDigest(d, cfg); + expect(text).toBe("*Headline*: fine."); + expect(d.post).toHaveBeenCalledWith("*Headline*: fine."); + + const f = facts(d); + expect(f.window.from).toBe("2026-11-12T12:00:00.000Z"); + expect(f.monitors).toEqual([{ name: "data-proxy: M7 Latency", enabled: true, description: "p99 > 5 s" }]); + // One event per notification: the webhook copy, not the duplicate email. + expect(f.alertNotifications).toEqual([ + { policy: "data-proxy: M7 Latency", sent: "2026-11-17T19:58:00Z", state: "firing", value: "9414" }, + { policy: "data-proxy: M7 Latency", sent: "2026-11-17T20:13:00Z", state: "resolved", value: "1571" }, + ]); + expect(f.workersIssues.map((i: any) => i.title)).toEqual(["HttpServerError · HTTP 520"]); + // Windows with too little traffic to mean anything are dropped. + expect(f.traffic.worstReadWindows15m).toEqual([{ bucket: "b", read_p99_ms: 3600, requests: "400000" }]); + expect(f.logs.topWarnErrorSources).toEqual([ + { level: "warn", message: "[WARN] request failed { request_id=…, duration_ms=…, path=/ftw/benchmark-data/… }", count: 15 }, + ]); + expect(f.openIssues).toEqual([{ number: 206, title: "Streamed PUTs intermittently 520" }]); + + const sql = (d.cf as any).mock.calls.filter((c: any[]) => c[0] === "/analytics_engine/sql").map((c: any[]) => c[1].body); + expect(sql.every((s: string) => s.includes("FROM source_data_proxy_production") && s.endsWith("FORMAT JSON"))).toBe(true); + }); + + it("reports a failing source as unavailable instead of failing the digest", async () => { + const d = deps({ listOpenIssues: vi.fn(async () => { throw new Error("403 Forbidden"); }) }); + await runDigest(d, cfg); + expect(facts(d).openIssues.unavailable).toContain("403 Forbidden"); + expect(d.post).toHaveBeenCalled(); + }); +}); + +describe("normalizeLogMessage", () => { + it("keeps the account and product of a path but drops per-request detail", () => { + expect(normalizeLogMessage("status=503, duration_ms=36.0, path=/a/b/c/d.tif, request_id=x1")).toBe( + "status=503, duration_ms=…, path=/a/b/…, request_id=…" + ); + }); +}); diff --git a/src/digest.ts b/src/digest.ts new file mode 100644 index 0000000..cffeed8 --- /dev/null +++ b/src/digest.ts @@ -0,0 +1,260 @@ +// src/digest.ts +// +// Weekly observability digest for the data proxy: gather a fixed set of facts +// from Cloudflare (alert history, Workers Issues, Analytics Engine, Workers +// Logs) and the repo's open issues, have Claude write a short review, and post +// it to Slack. It only suggests: it never files issues or changes monitors. +import { normalizeMessage } from "./fingerprint.js"; + +export interface DigestConfig { + repo: string; + script_name: string; + analytics_dataset: string; + days: number; +} + +export interface DigestDeps { + /** Cloudflare API call under /accounts/{id}. Returns the parsed JSON body. */ + cf(path: string, init?: { method?: string; body?: string; contentType?: string }): Promise; + accountId: string; + listOpenIssues(repo: string): Promise>; + summarize(system: string, prompt: string): Promise; + post(text: string): Promise; + now(): number; +} + +const SYSTEM_PROMPT = `You review a week of production observability for data.source.coop, Source Cooperative's Cloudflare Worker data proxy, and write a short digest for the team's #devops Slack channel. + +Write Slack mrkdwn, not GitHub markdown: *bold*, _italic_, • bullets, links as . Link issues as . At most about 2,500 characters. No preamble. + +Structure: +*Headline*: one or two sentences: was the week healthy, and what is the one thing to look at. +*Alerts*: each monitor that fired: how often, and whether it looks real (an incident visible in errors or latency) or noise (threshold or window too tight). Monitors that were silent need one line in total. Use the policy descriptions for what each one watches. +*Errors and latency*: the 5xx rate trend, which products/accounts drove errors, read p99 and its worst windows. Cite numbers. +*Noise*: the top WARN/ERROR log sources and Workers Issues. For each, the open issue that tracks it by number, or *untracked*. Ignore "Potential vulnerability scan" Issues unless their volume changed sharply. +*Suggested actions*: at most 5, most valuable first, each concrete (e.g. "file an issue for X", "raise M7's window to 30m"). These are suggestions for a human, not things you did. + +If a data source says "unavailable", say so in one line and work with the rest. Do not invent numbers.`; + +async function safe(fn: () => Promise): Promise { + try { + return await fn(); + } catch (e) { + return { unavailable: String(e).slice(0, 300) }; + } +} + +/** Rows from a SQL API response, whichever envelope it comes in. */ +function rows(r: any): any[] { + return r?.data ?? r?.result?.data ?? r?.result ?? []; +} + +/** Collapse per-request detail so one log line's variants group together. */ +export function normalizeLogMessage(message: string): string { + return normalizeMessage(message) + .replace(/request_id=[^,}\s]+/g, "request_id=…") + .replace(/duration_ms=[\d.]+/g, "duration_ms=…") + .replace(/(path=\/[^/\s,]+\/[^/\s,]+)\/[^\s,]*/g, "$1/…"); +} + +/** One entry per alert notification sent, preferring the webhook copy of each. */ +function alertEvents(history: any[]): Array<{ policy: string; sent: string; state: string; value: string | null }> { + const webhook = history.filter((h) => h.mechanism_type === "webhook"); + const chosen = webhook.length > 0 ? webhook : history; + return chosen.map((h) => { + const body = String(h.alert_body ?? ""); + return { + policy: h.name, + sent: h.sent, + state: /Alert State: Resolved/.test(body) ? "resolved" : "firing", + value: body.match(/Triggered Result: ([\d.]+)/)?.[1] ?? null, + }; + }); +} + +export async function gatherDigest(deps: DigestDeps, cfg: DigestConfig): Promise> { + const end = deps.now(); + const start = end - cfg.days * 24 * 3600 * 1000; + const startIso = new Date(start).toISOString(); + const aeStart = startIso.slice(0, 19).replace("T", " "); + const ds = cfg.analytics_dataset; + + const ae = (sql: string) => + deps.cf("/analytics_engine/sql", { method: "POST", body: `${sql} FORMAT JSON`, contentType: "text/plain" }).then(rows); + const logs = (sql: string) => + deps.cf("/analytics/sql", { method: "POST", body: JSON.stringify({ query: sql }) }).then(rows); + const logWhere = `accountTag = '${deps.accountId}' AND scriptName = '${cfg.script_name}' AND timestamp >= '${startIso}'`; + + const [policies, alerts, issues, daily, readP99, readWindows, top5xx, topProducts, logLevels, logTop, openIssues] = + await Promise.all([ + safe(async () => + ((await deps.cf("/alerting/v3/policies")).result ?? []) + .filter((p: any) => String(p.name).startsWith("data-proxy")) + .map((p: any) => ({ name: p.name, enabled: p.enabled, description: p.description })) + ), + safe(async () => { + const all: any[] = []; + for (let page = 1; page <= 5; page++) { + const r = await deps.cf(`/alerting/v3/history?since=${encodeURIComponent(startIso)}&per_page=100&page=${page}`); + const batch = r.result ?? []; + all.push(...batch); + if (batch.length < 100) break; + } + return alertEvents(all.filter((h) => String(h.name).startsWith("data-proxy"))); + }), + safe(async () => { + const r = await deps.cf(`/workers/observability/issues?service=${cfg.script_name}&perPage=100`); + const list: any[] = Array.isArray(r.result) ? r.result : r.result?.issues ?? []; + return list + .filter((i) => i.lastObserved >= start) + .map((i) => ({ + title: i.title, + type: i.type, + totalCount: i.count, + firstSeen: new Date(i.firstObserved).toISOString(), + lastSeen: new Date(i.lastObserved).toISOString(), + })); + }), + safe(() => + ae(`SELECT toStartOfDay(timestamp) AS day, SUM(_sample_interval) AS requests, SUM(IF(double2 >= 500, _sample_interval, 0)) AS errors_5xx, SUM(IF(blob4 = 'PUT', _sample_interval, 0)) AS puts FROM ${ds} WHERE timestamp >= toDateTime('${aeStart}') GROUP BY day ORDER BY day`) + ), + safe(() => + ae(`SELECT toStartOfDay(timestamp) AS day, quantileWeighted(0.99)(double3, _sample_interval) AS read_p99_ms FROM ${ds} WHERE timestamp >= toDateTime('${aeStart}') AND (blob4 = 'GET' OR blob4 = 'HEAD') GROUP BY day ORDER BY day`) + ), + safe(async () => + ( + await ae( + `SELECT toStartOfInterval(timestamp, INTERVAL '15' MINUTE) AS bucket, quantileWeighted(0.99)(double3, _sample_interval) AS read_p99_ms, SUM(_sample_interval) AS requests FROM ${ds} WHERE timestamp >= toDateTime('${aeStart}') AND (blob4 = 'GET' OR blob4 = 'HEAD') GROUP BY bucket` + ) + ) + .filter((w: any) => Number(w.requests) >= 1000) + .sort((a: any, b: any) => b.read_p99_ms - a.read_p99_ms) + .slice(0, 8) + ), + safe(() => + ae(`SELECT blob1 AS account, blob2 AS product, blob4 AS method, double2 AS status, SUM(_sample_interval) AS requests FROM ${ds} WHERE timestamp >= toDateTime('${aeStart}') AND double2 >= 500 GROUP BY account, product, method, status ORDER BY requests DESC LIMIT 20`) + ), + safe(() => + ae(`SELECT blob1 AS account, blob2 AS product, SUM(_sample_interval) AS requests FROM ${ds} WHERE timestamp >= toDateTime('${aeStart}') GROUP BY account, product ORDER BY requests DESC LIMIT 10`) + ), + safe(() => + logs(`SELECT toStartOfDay(timestamp) AS day, level, COUNT(*) AS n FROM logs.workersLogs WHERE ${logWhere} GROUP BY day, level ORDER BY day LIMIT 100`) + ), + safe(async () => { + const raw = await logs( + `SELECT level, substring(message, 1, 200) AS msg, COUNT(*) AS n FROM logs.workersLogs WHERE ${logWhere} AND (level = 'warn' OR level = 'error') GROUP BY level, msg ORDER BY n DESC LIMIT 500` + ); + const grouped = new Map(); + for (const r of raw) { + const message = normalizeLogMessage(String(r.msg)); + const key = `${r.level}|${message}`; + const g = grouped.get(key) ?? { level: r.level, message, count: 0 }; + g.count += Number(r.n); + grouped.set(key, g); + } + return [...grouped.values()].sort((a, b) => b.count - a.count).slice(0, 20); + }), + safe(() => deps.listOpenIssues(cfg.repo)), + ]); + + return { + window: { from: startIso, to: new Date(end).toISOString(), days: cfg.days }, + repo: cfg.repo, + monitors: policies, + alertNotifications: alerts, + workersIssues: issues, + traffic: { daily, readP99Daily: readP99, worstReadWindows15m: readWindows, top5xx, topProducts }, + logs: { levelsByDay: logLevels, topWarnErrorSources: logTop }, + openIssues, + }; +} + +export async function runDigest(deps: DigestDeps, cfg: DigestConfig): Promise { + const facts = await gatherDigest(deps, cfg); + const prompt = `Repository: ${cfg.repo}\n\nThe week's data as JSON:\n\n${JSON.stringify(facts, null, 1)}`; + const text = await deps.summarize(SYSTEM_PROMPT, prompt); + await deps.post(text); + return text; +} + +// CLI entry point -- called by GitHub Actions +async function main() { + const { loadConfig } = await import("./config.js"); + const { GitHubClient } = await import("./github.js"); + const { query } = await import("@anthropic-ai/claude-agent-sdk"); + + const config = loadConfig(); + if (!config.digest) throw new Error("config.yaml has no digest section"); + const cfg = { ...config.digest, days: Number(process.env.DIGEST_DAYS || config.digest.days) }; + const accountId = requireEnv("CLOUDFLARE_ACCOUNT_ID"); + const apiToken = requireEnv("CLOUDFLARE_API_TOKEN"); + const webhook = process.env.SLACK_WEBHOOK_URL; + + const github = new GitHubClient({ + appId: requireEnv("SC_DEV_BOT_APP_ID"), + privateKey: requireEnv("SC_DEV_BOT_PRIVATE_KEY"), + installationId: requireEnv("SC_DEV_BOT_INSTALLATION_ID"), + }); + + const text = await runDigest( + { + accountId, + now: () => Date.now(), + async cf(path, init = {}) { + const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${accountId}${path}`, { + method: init.method ?? "GET", + headers: { + Authorization: `Bearer ${apiToken}`, + ...(init.body ? { "Content-Type": init.contentType ?? "application/json" } : {}), + }, + body: init.body, + }); + if (!response.ok) throw new Error(`${init.method ?? "GET"} ${path}: ${response.status} ${(await response.text()).slice(0, 200)}`); + return response.json(); + }, + listOpenIssues: (repo) => github.listOpenIssues(repo), + async summarize(systemPrompt, prompt) { + let result: string | null = null; + for await (const message of query({ + prompt, + options: { model: config.anthropic_model, systemPrompt, maxTurns: 1, tools: [], settingSources: [] }, + })) { + if (message.type === "result") { + if (message.subtype !== "success") throw new Error(`Digest query failed: ${message.subtype}`); + result = message.result; + } + } + if (!result) throw new Error("Digest query returned no result"); + return result.trim(); + }, + async post(text) { + if (!webhook) { + console.log("SLACK_WEBHOOK_URL not set; not posting."); + return; + } + const response = await fetch(webhook, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text }), + }); + if (!response.ok) throw new Error(`Slack webhook: ${response.status} ${await response.text()}`); + }, + }, + cfg + ); + console.log(text); +} + +function requireEnv(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`Missing required environment variable: ${name}`); + return value; +} + +const isDirectRun = process.argv[1]?.endsWith("digest.ts") || process.argv[1]?.endsWith("digest.js"); +if (isDirectRun) { + main().catch((err) => { + console.error("Digest failed:", err); + process.exit(1); + }); +} diff --git a/src/github.ts b/src/github.ts index fb6a569..314621b 100644 --- a/src/github.ts +++ b/src/github.ts @@ -125,4 +125,13 @@ export class GitHubClient { inputs, }); } + + /** Open issues (not pull requests), most recently updated first, up to 100. */ + async listOpenIssues(repo: string): Promise> { + const items = (await this.request( + "GET", + `/repos/${repo}/issues?state=open&per_page=100&sort=updated` + )) as Array<{ number: number; title: string; pull_request?: unknown }>; + return items.filter((i) => !i.pull_request).map((i) => ({ number: i.number, title: i.title })); + } } diff --git a/src/index.test.ts b/src/index.test.ts index 72cefea..d4942ab 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -125,4 +125,21 @@ describe("runMonitor", () => { await runMonitor(mockDeps); expect(mockDeps.github.commentOnIssue).toHaveBeenCalled(); }); + + it("reports a running total from Cloudflare Issues as a total, and raises total_count to it", async () => { + const fp = computeFingerprint("HttpServerError · HTTP 520", null, 520); + (mockDeps.cloudflareSource.fetchErrors as any).mockResolvedValueOnce([ + { message: "HttpServerError · HTTP 520", stackLocation: null, httpStatus: 520, source: "source-cooperative/data.source.coop", releaseVersion: "unknown", timestamp: Date.now(), dashboardUrl: null, rawLog: null, count: 9754, countIsTotal: true }, + ]); + (mockDeps.d1.query as any).mockResolvedValueOnce([ + { fingerprint: fp, repo: "source-cooperative/data.source.coop", github_issue_number: 7, github_issue_state: "open", release_versions: '["unknown"]', last_commented_at: null }, + ]); + + await runMonitor(mockDeps); + const comment = (mockDeps.github.commentOnIssue as any).mock.calls[0][2] as string; + expect(comment).toContain("9754 times in total"); + const update = (mockDeps.d1.execute as any).mock.calls.find((c: any[]) => String(c[0]).startsWith("UPDATE errors")); + expect(update[0]).toContain("total_count = MAX(total_count + ?, ?)"); + expect(update[1].slice(2, 4)).toEqual([0, 9754]); + }); }); diff --git a/src/index.ts b/src/index.ts index 965cd43..de13a62 100644 --- a/src/index.ts +++ b/src/index.ts @@ -55,6 +55,17 @@ function formatSampleLogs(logs: string[]): string { return `\n\n${details}`; } +function describeCount(count: number, countIsTotal: boolean, windowHours: number): string { + return countIsTotal + ? `${count} times in total since it was first seen` + : `${count} times in the last ${windowHours} hours`; +} + +/** Params for `total_count = MAX(total_count + ?, ?)`: add a window's count, or raise to a running total. */ +function totalParams(data: { count: number; countIsTotal: boolean }): [number, number] { + return data.countIsTotal ? [0, data.count] : [data.count, 0]; +} + function formatVersion(version: string, repo: string): string { if (version === "unknown") return "`unknown`"; // Git SHA (7-40 hex chars) → link to commit @@ -108,14 +119,14 @@ export async function runMonitor(deps: MonitorDeps): Promise { const maxSamples = 3; const errorsByFingerprint = new Map< string, - { error: RawError; count: number; fingerprint: string; releaseVersions: Set; sampleUrls: string[]; sampleLogs: string[] } + { error: RawError; count: number; countIsTotal: boolean; fingerprint: string; releaseVersions: Set; sampleUrls: string[]; sampleLogs: string[] } >(); for (const error of allErrors) { const fp = computeFingerprint(error.message, error.stackLocation, error.httpStatus); const existing = errorsByFingerprint.get(fp); if (existing) { - existing.count++; + existing.count += error.count ?? 1; existing.releaseVersions.add(error.releaseVersion); if (existing.sampleUrls.length < maxSamples) { if (error.dashboardUrl) existing.sampleUrls.push(error.dashboardUrl); @@ -124,7 +135,8 @@ export async function runMonitor(deps: MonitorDeps): Promise { } else { errorsByFingerprint.set(fp, { error, - count: 1, + count: error.count ?? 1, + countIsTotal: error.countIsTotal ?? false, fingerprint: fp, releaseVersions: new Set([error.releaseVersion]), sampleUrls: error.dashboardUrl ? [error.dashboardUrl] : [], @@ -177,21 +189,21 @@ export async function runMonitor(deps: MonitorDeps): Promise { if (newVersions.length > 0 && known.github_issue_number) { console.log(` Regression: ${fp.slice(0, 8)} on ${newVersions.join(", ")} (issue #${known.github_issue_number})`); - const comment = `\u26a0\ufe0f **Regression detected**\n\nThis error reappeared on ${formatVersions(newVersions, known.repo)} (${data.count} occurrences in the last ${windowHours} hours).\n\nEither the fix didn't address this case or there was a regression.${formatSampleLinks(data.sampleUrls)}${formatSampleLogs(data.sampleLogs)}`; + const comment = `\u26a0\ufe0f **Regression detected**\n\nThis error reappeared on ${formatVersions(newVersions, known.repo)} (${describeCount(data.count, data.countIsTotal, windowHours)}).\n\nEither the fix didn't address this case or there was a regression.${formatSampleLinks(data.sampleUrls)}${formatSampleLogs(data.sampleLogs)}`; await deps.github.reopenIssue(known.repo, known.github_issue_number); await deps.github.commentOnIssue(known.repo, known.github_issue_number, comment); issuesReopened++; const allVersions = [...knownVersions, ...newVersions]; await deps.d1.execute( - "UPDATE errors SET github_issue_state = 'open', release_versions = ?, last_seen_at = ?, total_count = total_count + ?, window_count = ?, updated_at = ? WHERE fingerprint = ?", - [JSON.stringify(allVersions), now, data.count, data.count, now, fp] + "UPDATE errors SET github_issue_state = 'open', release_versions = ?, last_seen_at = ?, total_count = MAX(total_count + ?, ?), window_count = ?, updated_at = ? WHERE fingerprint = ?", + [JSON.stringify(allVersions), now, ...totalParams(data), data.count, now, fp] ); } else { // Closed issue recurring on an already-known release — keep counts current await deps.d1.execute( - "UPDATE errors SET last_seen_at = ?, total_count = total_count + ?, window_count = ?, updated_at = ? WHERE fingerprint = ?", - [now, data.count, data.count, now, fp] + "UPDATE errors SET last_seen_at = ?, total_count = MAX(total_count + ?, ?), window_count = ?, updated_at = ? WHERE fingerprint = ?", + [now, ...totalParams(data), data.count, now, fp] ); } } else if (known.github_issue_state === "open" && known.github_issue_number) { @@ -200,18 +212,18 @@ export async function runMonitor(deps: MonitorDeps): Promise { if (daysSinceComment >= deps.config.comment_cadence_days) { console.log(` Weekly comment: ${fp.slice(0, 8)} (issue #${known.github_issue_number}, ${data.count} occurrences)`); - const comment = `\ud83d\udcca **Ongoing error report**\n\nThis error occurred ${data.count} times in the last ${windowHours} hours.\n\nRelease version(s): ${formatVersions(data.releaseVersions, known.repo)}${formatSampleLinks(data.sampleUrls)}${formatSampleLogs(data.sampleLogs)}`; + const comment = `\ud83d\udcca **Ongoing error report**\n\nThis error occurred ${describeCount(data.count, data.countIsTotal, windowHours)}.\n\nRelease version(s): ${formatVersions(data.releaseVersions, known.repo)}${formatSampleLinks(data.sampleUrls)}${formatSampleLogs(data.sampleLogs)}`; await deps.github.commentOnIssue(known.repo, known.github_issue_number, comment); issuesCommented++; await deps.d1.execute( - "UPDATE errors SET last_commented_at = ?, last_seen_at = ?, total_count = total_count + ?, window_count = ?, updated_at = ? WHERE fingerprint = ?", - [now, now, data.count, data.count, now, fp] + "UPDATE errors SET last_commented_at = ?, last_seen_at = ?, total_count = MAX(total_count + ?, ?), window_count = ?, updated_at = ? WHERE fingerprint = ?", + [now, now, ...totalParams(data), data.count, now, fp] ); } else { await deps.d1.execute( - "UPDATE errors SET last_seen_at = ?, total_count = total_count + ?, window_count = ?, updated_at = ? WHERE fingerprint = ?", - [now, data.count, data.count, now, fp] + "UPDATE errors SET last_seen_at = ?, total_count = MAX(total_count + ?, ?), window_count = ?, updated_at = ? WHERE fingerprint = ?", + [now, ...totalParams(data), data.count, now, fp] ); } } @@ -328,6 +340,7 @@ async function main() { accountId: requireEnv("CLOUDFLARE_ACCOUNT_ID"), scriptName: cfRepo?.cloudflare_script_name ?? "", dashboardUrl: cfRepo?.dashboard_url, + issuesSince: cfRepo?.cloudflare_issues_since, }); const classifier = new ErrorClassifier({ diff --git a/src/sources/cloudflare.test.ts b/src/sources/cloudflare.test.ts index c301585..06feeec 100644 --- a/src/sources/cloudflare.test.ts +++ b/src/sources/cloudflare.test.ts @@ -4,64 +4,66 @@ import { CloudflareSource } from "./cloudflare.js"; const mockFetch = vi.fn(); global.fetch = mockFetch; +const now = Date.now(); +const issue = (over: Record) => ({ + id: "i1", + service: "source-data-proxy", + title: "HttpServerError · HTTP 520", + type: "exception", + status: "active", + count: 42, + firstObserved: now - 60_000, + lastObserved: now - 1_000, + ...over, +}); +const page = (issues: unknown[]) => ({ ok: true, json: async () => ({ success: true, result: issues }) }); + describe("CloudflareSource", () => { const source = new CloudflareSource({ apiToken: "test-token", accountId: "test-account", - scriptName: "data-source-coop", + scriptName: "source-data-proxy", + issuesSince: new Date(now - 3_600_000).toISOString(), }); - beforeEach(() => { - mockFetch.mockReset(); - }); - - it("fetches error events from Workers Observability API", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ - success: true, - result: { - events: { - events: [ - { - "$metadata": { - id: "evt1", - error: "connection refused", - level: "error", - message: "Failed to fetch upstream", - service: "data-source-coop", - statusCode: 502, - startTime: Date.now() - 1000, - }, - timestamp: Date.now() - 1000, - }, - ], - }, - }, - }), - }); + beforeEach(() => mockFetch.mockReset()); + it("maps a Workers Issue to an error carrying its running total", async () => { + mockFetch.mockResolvedValueOnce(page([issue({})])); const errors = await source.fetchErrors(6); expect(errors).toHaveLength(1); - expect(errors[0].message).toContain("Failed to fetch upstream"); - expect(errors[0].httpStatus).toBe(502); - expect(errors[0].source).toBe("source-cooperative/data.source.coop"); - }); - - it("sends correct query with error filter", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ success: true, result: { events: { events: [] } } }), + expect(errors[0]).toMatchObject({ + message: "HttpServerError · HTTP 520", + httpStatus: 520, + count: 42, + countIsTotal: true, + source: "source-cooperative/data.source.coop", }); + const url = new URL(mockFetch.mock.calls[0][0]); + expect(url.pathname).toContain("/workers/observability/issues"); + expect(url.searchParams.get("service")).toBe("source-data-proxy"); + }); - await source.fetchErrors(6); - - const call = mockFetch.mock.calls[0]; - const body = JSON.parse(call[1].body); - expect(body.parameters.filters).toEqual( - expect.arrayContaining([ - expect.objectContaining({ key: "$metadata.level", operation: "eq", value: "error" }), + it("skips scans, other services, stale groups and groups older than issuesSince", async () => { + mockFetch.mockResolvedValueOnce( + page([ + issue({ id: "scan", type: "security", title: "Potential vulnerability scan" }), + issue({ id: "other", service: "something-else" }), + issue({ id: "stale", lastObserved: now - 7 * 3_600_000 }), + issue({ id: "old", firstObserved: now - 2 * 3_600_000 }), + issue({ id: "keep", title: "[ERROR] backend STS transport error" }), ]) ); + const errors = await source.fetchErrors(6); + expect(errors.map((e) => e.message)).toEqual(["[ERROR] backend STS transport error"]); + expect(errors[0].httpStatus).toBeNull(); + }); + + it("pages until a short page", async () => { + const full = Array.from({ length: 100 }, (_, n) => issue({ id: `i${n}`, title: `E${n}` })); + mockFetch.mockResolvedValueOnce(page(full)).mockResolvedValueOnce(page([issue({ id: "last" })])); + const errors = await source.fetchErrors(6); + expect(errors).toHaveLength(101); + expect(mockFetch).toHaveBeenCalledTimes(2); }); }); diff --git a/src/sources/cloudflare.ts b/src/sources/cloudflare.ts index 20f90ed..b49fe50 100644 --- a/src/sources/cloudflare.ts +++ b/src/sources/cloudflare.ts @@ -6,23 +6,34 @@ interface CloudflareConfig { accountId: string; scriptName: string; dashboardUrl?: string; + /** Groups Cloudflare first saw before this are left alone: they predate sourcebot watching them. */ + issuesSince?: string; } -interface ObservabilityEvent { - $metadata: { - id: string; - error?: string; - level: string; - message?: string; - service: string; - statusCode?: number; - startTime?: number; - endTime?: number; - traceId?: string; - }; - timestamp: number; +/** One Workers Observability Issue: Cloudflare's own grouping of a recurring error. */ +interface WorkersIssue { + id: string; + service: string; + title: string; + type: string; + status: string; + count: number; + firstObserved: number; + lastObserved: number; } +const PER_PAGE = 100; +const MAX_PAGES = 10; + +/** + * Reads Cloudflare's Workers Observability Issues rather than raw ERROR events. + * Cloudflare already groups every exception, 5xx and ERROR line into Issues, so + * this sees every group no matter how many lines it logs, where a raw-event + * query caps out at a few hundred lines that one noisy error can fill. + * + * Issues only carry a running total, so each error's `count` is that total + * (`countIsTotal`), not a per-window count. + */ export class CloudflareSource { private config: CloudflareConfig; private repoName = "source-cooperative/data.source.coop"; @@ -32,76 +43,63 @@ export class CloudflareSource { } async fetchErrors(windowHours: number): Promise { - const now = Date.now(); - const from = now - windowHours * 60 * 60 * 1000; + const from = Date.now() - windowHours * 60 * 60 * 1000; + const since = this.config.issuesSince ? Date.parse(this.config.issuesSince) : 0; - const response = await fetch( - `https://api.cloudflare.com/client/v4/accounts/${this.config.accountId}/workers/observability/telemetry/query`, - { - method: "POST", - headers: { - Authorization: `Bearer ${this.config.apiToken}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - queryId: "sourcebot-errors", - timeframe: { from, to: now }, - view: "events", - limit: 500, - dry: true, - parameters: { - datasets: [], - filters: [ - { key: "$metadata.level", operation: "eq", type: "string", value: "error" }, - { key: "$metadata.service", operation: "eq", type: "string", value: this.config.scriptName }, - ], - filterCombination: "and", - }, - }), + const issues: WorkersIssue[] = []; + for (let page = 1; page <= MAX_PAGES; page++) { + const url = new URL( + `https://api.cloudflare.com/client/v4/accounts/${this.config.accountId}/workers/observability/issues` + ); + url.searchParams.set("service", this.config.scriptName); + url.searchParams.set("page", String(page)); + url.searchParams.set("perPage", String(PER_PAGE)); + const response = await fetch(url, { + headers: { Authorization: `Bearer ${this.config.apiToken}` }, + }); + if (!response.ok) { + throw new Error(`CF Workers Issues API error: ${response.status} ${await response.text()}`); } - ); - - if (!response.ok) { - throw new Error(`CF Observability API error: ${response.status} ${await response.text()}`); - } - - const data = (await response.json()) as { - success: boolean; - result?: { events?: ObservabilityEvent[] | { events?: ObservabilityEvent[] } }; - }; - - if (!data.success) { - throw new Error(`CF Observability query failed: ${JSON.stringify(data)}`); + const data = (await response.json()) as { + success: boolean; + result?: WorkersIssue[] | { issues?: WorkersIssue[] }; + }; + if (!data.success) { + throw new Error(`CF Workers Issues query failed: ${JSON.stringify(data).slice(0, 500)}`); + } + const batch = Array.isArray(data.result) ? data.result : data.result?.issues ?? []; + issues.push(...batch); + if (batch.length < PER_PAGE) break; } - // The Observability telemetry/query response has shifted shape across versions; - // `result.events` may be either the array directly or `{ events: [...] }`. - const eventsField = data.result?.events; - const events: ObservabilityEvent[] = Array.isArray(eventsField) - ? eventsField - : eventsField?.events ?? []; - - return events.map((event) => { - const message = event.$metadata.message ?? event.$metadata.error ?? "Unknown error"; - const time = new Date(event.timestamp).toISOString(); - const rawLog = [ - `${time} [${event.$metadata.level}] ${event.$metadata.service}`, - event.$metadata.statusCode ? `Status: ${event.$metadata.statusCode}` : null, - event.$metadata.traceId ? `Trace: ${event.$metadata.traceId}` : null, - "", - message, - ].filter(Boolean).join("\n"); - - return { - message, - stackLocation: null, - httpStatus: event.$metadata.statusCode ?? null, - source: this.repoName, - releaseVersion: "unknown", - timestamp: event.timestamp, - dashboardUrl: this.config.dashboardUrl ?? null, - rawLog: redactLog(rawLog), - }; - }); + return issues + .filter((i) => i.service === this.config.scriptName) + // Vulnerability-scan detections are bots probing for /.env and the like, + // not defects in the proxy. + .filter((i) => i.type !== "security") + .filter((i) => i.lastObserved >= from && i.firstObserved >= since) + .map((i) => { + const rawLog = [ + `Workers Issue ${i.id} (${i.type}, ${i.status}) on ${i.service}`, + `First seen: ${new Date(i.firstObserved).toISOString()}`, + `Last seen: ${new Date(i.lastObserved).toISOString()}`, + `Occurrences: ${i.count}`, + "", + i.title, + ].join("\n"); + const status = i.title.match(/\bHTTP (\d{3})\b/); + return { + message: i.title, + stackLocation: null, + httpStatus: status ? Number(status[1]) : null, + source: this.repoName, + releaseVersion: "unknown", + timestamp: i.lastObserved, + dashboardUrl: this.config.dashboardUrl ?? null, + rawLog: redactLog(rawLog), + count: i.count, + countIsTotal: true, + }; + }); } } diff --git a/src/sources/vercel.ts b/src/sources/vercel.ts index 591f24d..5ac9d34 100644 --- a/src/sources/vercel.ts +++ b/src/sources/vercel.ts @@ -9,6 +9,10 @@ export interface RawError { timestamp: number; dashboardUrl: string | null; rawLog: string | null; + /** Occurrences this entry stands for (default 1). */ + count?: number; + /** `count` is a running total since the error was first seen, not a count for this window. */ + countIsTotal?: boolean; } interface VercelConfig {