Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions .github/workflows/digest.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
name: Observability Digest

on:
schedule:
- cron: "0 16 * * 1" # Mondays 16:00 UTC
workflow_dispatch:
inputs:
days:
description: "Days to review (default: 7)"
required: false
type: number
post:
description: "Post to Slack (off prints the digest to the log only)"
required: false
type: boolean
default: true

permissions:
contents: read
actions: write # keepalive below

jobs:
digest:
runs-on: ubuntu-latest
timeout-minutes: 10

steps:
- name: Checkout sourcebot
uses: actions/checkout@v4

- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: "22"
cache: "npm"

- name: Install dependencies
run: npm ci

- name: Run digest
run: npx tsx src/digest.ts
env:
CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
SC_DEV_BOT_APP_ID: ${{ vars.SC_DEV_BOT_APP_ID }}
SC_DEV_BOT_PRIVATE_KEY: ${{ secrets.SC_DEV_BOT_PRIVATE_KEY }}
SC_DEV_BOT_INSTALLATION_ID: ${{ vars.SC_DEV_BOT_INSTALLATION_ID }}
SLACK_WEBHOOK_URL: ${{ (github.event_name == 'schedule' || inputs.post) && secrets.SLACK_WEBHOOK_URL || '' }}
DIGEST_DAYS: ${{ inputs.days }}

# GitHub disables scheduled workflows after 60 days without repository
# activity (that stopped the monitor on 2026-10-10). Re-enabling them from
# a scheduled run keeps them on.
- name: Keep scheduled workflows enabled
if: always()
run: |
gh workflow enable monitor.yml -R "$GITHUB_REPOSITORY"
gh workflow enable digest.yml -R "$GITHUB_REPOSITORY"
env:
GH_TOKEN: ${{ github.token }}
15 changes: 15 additions & 0 deletions .github/workflows/monitor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ on:
required: false
type: number

permissions:
contents: read
actions: write # keepalive below

jobs:
monitor:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -41,3 +45,14 @@ jobs:
SC_DEV_BOT_PRIVATE_KEY: ${{ secrets.SC_DEV_BOT_PRIVATE_KEY }}
SC_DEV_BOT_INSTALLATION_ID: ${{ vars.SC_DEV_BOT_INSTALLATION_ID }}
WINDOW_HOURS: ${{ inputs.window_hours }}

# GitHub disables scheduled workflows after 60 days without repository
# activity (that stopped this monitor on 2026-10-10). Re-enabling them from
# a scheduled run keeps them on.
- name: Keep scheduled workflows enabled
if: always()
run: |
gh workflow enable monitor.yml -R "$GITHUB_REPOSITORY"
gh workflow enable digest.yml -R "$GITHUB_REPOSITORY"
env:
GH_TOKEN: ${{ github.token }}
20 changes: 17 additions & 3 deletions DEVELOPMENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ At https://dash.cloudflare.com/profile/api-tokens, create a Custom Token with:
- Account → D1: Edit
- Account → Workers Observability: Read
- Account → Account Analytics: Read
- Account → Notifications: Read (the digest reads alert policies and history)

Scope to your specific account.

Expand Down Expand Up @@ -67,6 +68,9 @@ At https://console.anthropic.com/settings/keys, create a key named `sourcebot`.
| Variable | `VERCEL_TEAM_ID` |
| Secret | `CLOUDFLARE_API_TOKEN` |
| Secret | `VERCEL_API_TOKEN` |
| Secret | `SLACK_WEBHOOK_URL` |

`SLACK_WEBHOOK_URL` is a Slack incoming webhook for the channel the weekly digest posts to (#devops). Without it the digest prints to the workflow log instead of posting.

### 7. Initialize the D1 Schema

Expand Down Expand Up @@ -98,12 +102,19 @@ repos:

- name: source-cooperative/data.source.coop
log_source: cloudflare_workers
cloudflare_script_name: data-source-coop
auto_fix: true
cloudflare_script_name: source-data-proxy
cloudflare_issues_since: "2026-11-19T00:00:00Z" # older Workers Issues groups are not filed
auto_fix: false

schedule: "0 */6 * * *"
comment_cadence_days: 7
anthropic_model: claude-sonnet-4-6

digest:
repo: source-cooperative/data.source.coop
script_name: source-data-proxy
analytics_dataset: source_data_proxy_production
days: 7
```

When `auto_fix: true`, the monitor self-assigns new issues to Scooter, triggering the fix workflow automatically.
Expand All @@ -115,6 +126,7 @@ npm install
npm test # Run test suite (vitest)
npm run lint # Type check (tsc --noEmit)
npm run monitor # Run the monitor locally (requires env vars)
npm run digest # Run the digest locally; prints instead of posting unless SLACK_WEBHOOK_URL is set
```

## Project Structure
Expand All @@ -123,6 +135,7 @@ npm run monitor # Run the monitor locally (requires env vars)
sourcebot/
├── .github/workflows/
│ ├── monitor.yml Cron monitor
│ ├── digest.yml Weekly observability digest
│ ├── fix.yml Reusable fix workflow
│ └── ci.yml Test and type-check on PRs
├── caller-workflows/
Expand All @@ -131,14 +144,15 @@ sourcebot/
│ └── init-d1.sh One-time schema setup
├── src/
│ ├── index.ts Main orchestrator + CLI entry
│ ├── digest.ts Weekly observability digest + CLI entry
│ ├── config.ts Load config.yaml
│ ├── d1.ts D1 REST API client
│ ├── github.ts GitHub App auth + issue management
│ ├── fingerprint.ts Error normalization and hashing
│ ├── classifier.ts Anthropic API for error grouping
│ └── sources/
│ ├── vercel.ts Vercel runtime logs
│ └── cloudflare.ts Workers Observability
│ └── cloudflare.ts Workers Observability Issues
├── config.yaml Which repos to monitor
└── schema.sql D1 schema
```
11 changes: 10 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,20 @@ The GitHub App is named **Scooter** (Source Cooperative Operator for Observation
**Monitor workflow** (runs every 6 hours):

- Fetches runtime errors from Vercel for `source.coop`
- Fetches error-level logs from Cloudflare Workers Observability for `data.source.coop`
- Fetches Cloudflare's Workers Observability Issues (its own grouping of exceptions, 5xx responses and ERROR lines) for `data.source.coop`, skipping vulnerability-scan detections
- Computes a fingerprint for each error (normalized message + stack location + status code)
- Groups new errors via the Anthropic API and creates GitHub issues in the appropriate target repo
- Reopens closed issues when the same error recurs on a newer release version
- Comments weekly on open issues with updated occurrence counts

**Digest workflow** (Mondays 16:00 UTC):

- Gathers the past week of the data proxy's observability: alert notifications and monitor definitions, Workers Issues, Analytics Engine traffic (daily requests and 5xx, read p99 and its worst 15-minute windows, top 5xx and busiest products), Workers Logs WARN/ERROR volume and top sources, and `data.source.coop`'s open issues
- Has Claude write a short review (which alerts were real, what drove errors and latency, which noise sources are tracked by an open issue and which are not, suggested actions) and posts it to Slack
- Only suggests: it files no issues and changes no monitors

Both scheduled workflows re-enable themselves on each run, so GitHub's 60-day inactivity rule does not switch them off.

**Fix workflow** (triggered by issue assignment):

- Runs when a human assigns an issue to `scooter[bot]`
Expand All @@ -29,6 +37,7 @@ State lives in Cloudflare D1, accessed via its REST API.
```
GitHub Actions (this repo)
├── monitor.yml Cron: fetch logs → classify → create/update issues
├── digest.yml Weekly: gather proxy observability → summarize → Slack
└── fix.yml Reusable: called from target repos on issue assignment

Target repos (source.coop, data.source.coop)
Expand Down
16 changes: 14 additions & 2 deletions config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,23 @@ repos:

- name: source-cooperative/data.source.coop
log_source: cloudflare_workers
cloudflare_script_name: data-source-coop
cloudflare_script_name: source-data-proxy
# Workers Issues groups that already existed when sourcebot started reading
# Issues are tracked by hand (e.g. data.source.coop#206, #281); only file new ones.
cloudflare_issues_since: "2026-11-19T00:00:00Z"
dashboard_url: https://dash.cloudflare.com/1335bb5b87a1c2e1de2ca03408c23b71/workers/services/view/source-data-proxy/production/observability/invocations
auto_fix: true
# Most proxy error groups are infrastructure (Cloudflare 52x, backend STS),
# not code a bot can fix; a human assigns Scooter when one is.
auto_fix: false

schedule: "0 */6 * * *"
window_hours: 6
comment_cadence_days: 7
anthropic_model: claude-sonnet-4-6

# Weekly digest of the data proxy's observability, posted to Slack (src/digest.ts).
digest:
repo: source-cooperative/data.source.coop
script_name: source-data-proxy
analytics_dataset: source_data_proxy_production
days: 7
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"type": "module",
"scripts": {
"monitor": "tsx src/index.ts",
"digest": "tsx src/digest.ts",
"test": "vitest run",
"test:watch": "vitest",
"lint": "tsc --noEmit"
Expand Down
2 changes: 1 addition & 1 deletion src/classifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ export class ErrorClassifier {
HTTP Status: ${e.httpStatus ?? "N/A"}
Repo: ${e.source}
Release: ${e.releaseVersion}
Occurrences (last window): ${e.count}`
Occurrences: ${e.count}`
)
.join("\n\n");

Expand Down
4 changes: 4 additions & 0 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,16 @@ export interface RepoConfig {
log_source: "vercel" | "cloudflare_workers";
vercel_project_id?: string;
cloudflare_script_name?: string;
/** Workers Issues groups first seen before this ISO date are not filed. */
cloudflare_issues_since?: string;
dashboard_url?: string;
auto_fix: boolean;
}

export interface Config {
repos: RepoConfig[];
/** Weekly observability digest posted to Slack (src/digest.ts). */
digest?: { repo: string; script_name: string; analytics_dataset: string; days: number };
schedule: string;
comment_cadence_days: number;
anthropic_model: string;
Expand Down
99 changes: 99 additions & 0 deletions src/digest.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
import { describe, it, expect, vi } from "vitest";
import { runDigest, normalizeLogMessage, type DigestDeps } from "./digest.js";

const cfg = {
repo: "source-cooperative/data.source.coop",
script_name: "source-data-proxy",
analytics_dataset: "source_data_proxy_production",
days: 7,
};
const NOW = Date.parse("2026-11-19T12:00:00Z");

function deps(over: Partial<DigestDeps> = {}): DigestDeps {
return {
accountId: "acct",
now: () => NOW,
cf: vi.fn(async (path: string, init?: { body?: string }) => {
if (path.startsWith("/alerting/v3/policies"))
return { result: [{ name: "data-proxy: M7 Latency", enabled: true, description: "p99 > 5 s" }, { name: "other", enabled: true }] };
if (path.startsWith("/alerting/v3/history"))
return {
result: [
{ name: "data-proxy: M7 Latency", sent: "2026-11-17T19:58:00Z", mechanism_type: "webhook", alert_body: "Notification Policy: data-proxy: M7 Latency\nTriggered Result: 9414" },
{ name: "data-proxy: M7 Latency", sent: "2026-11-17T19:58:00Z", mechanism_type: "email", alert_body: "{}" },
{ name: "data-proxy: M7 Latency", sent: "2026-11-17T20:13:00Z", mechanism_type: "webhook", alert_body: "Alert State: Resolved\nTriggered Result: 1571" },
],
};
if (path.startsWith("/workers/observability/issues"))
return { result: [{ title: "HttpServerError · HTTP 520", type: "exception", count: 9754, firstObserved: NOW - 9e8, lastObserved: NOW - 1000 }, { title: "old", count: 1, firstObserved: 0, lastObserved: 0 }] };
if (path === "/analytics/sql") {
if (init?.body?.includes("substring(message"))
return {
data: [
{ level: "warn", msg: "[WARN] request failed { request_id=aaa, duration_ms=37.0, path=/ftw/benchmark-data/a/b.webp }", n: 10 },
{ level: "warn", msg: "[WARN] request failed { request_id=bbb, duration_ms=12.0, path=/ftw/benchmark-data/c.webp }", n: 5 },
],
};
return { data: [{ day: "2026-11-18", level: "warn", n: 40000 }] };
}
if (path === "/analytics_engine/sql") {
if (init?.body?.includes("AS bucket"))
return { data: [{ bucket: "a", read_p99_ms: 9000, requests: "10" }, { bucket: "b", read_p99_ms: 3600, requests: "400000" }] };
return { data: [{ day: "2026-11-18", requests: "30000000" }] };
}
throw new Error(`unexpected ${path}`);
}),
listOpenIssues: vi.fn(async () => [{ number: 206, title: "Streamed PUTs intermittently 520" }]),
summarize: vi.fn(async () => "*Headline*: fine."),
post: vi.fn(async () => undefined),
...over,
};
}

function facts(d: DigestDeps): any {
const prompt = (d.summarize as any).mock.calls[0][1] as string;
return JSON.parse(prompt.slice(prompt.indexOf("{")));
}

describe("runDigest", () => {
it("gathers the week's data, has it summarized, and posts the summary", async () => {
const d = deps();
const text = await runDigest(d, cfg);
expect(text).toBe("*Headline*: fine.");
expect(d.post).toHaveBeenCalledWith("*Headline*: fine.");

const f = facts(d);
expect(f.window.from).toBe("2026-11-12T12:00:00.000Z");
expect(f.monitors).toEqual([{ name: "data-proxy: M7 Latency", enabled: true, description: "p99 > 5 s" }]);
// One event per notification: the webhook copy, not the duplicate email.
expect(f.alertNotifications).toEqual([
{ policy: "data-proxy: M7 Latency", sent: "2026-11-17T19:58:00Z", state: "firing", value: "9414" },
{ policy: "data-proxy: M7 Latency", sent: "2026-11-17T20:13:00Z", state: "resolved", value: "1571" },
]);
expect(f.workersIssues.map((i: any) => i.title)).toEqual(["HttpServerError · HTTP 520"]);
// Windows with too little traffic to mean anything are dropped.
expect(f.traffic.worstReadWindows15m).toEqual([{ bucket: "b", read_p99_ms: 3600, requests: "400000" }]);
expect(f.logs.topWarnErrorSources).toEqual([
{ level: "warn", message: "[WARN] request failed { request_id=…, duration_ms=…, path=/ftw/benchmark-data/… }", count: 15 },
]);
expect(f.openIssues).toEqual([{ number: 206, title: "Streamed PUTs intermittently 520" }]);

const sql = (d.cf as any).mock.calls.filter((c: any[]) => c[0] === "/analytics_engine/sql").map((c: any[]) => c[1].body);
expect(sql.every((s: string) => s.includes("FROM source_data_proxy_production") && s.endsWith("FORMAT JSON"))).toBe(true);
});

it("reports a failing source as unavailable instead of failing the digest", async () => {
const d = deps({ listOpenIssues: vi.fn(async () => { throw new Error("403 Forbidden"); }) });
await runDigest(d, cfg);
expect(facts(d).openIssues.unavailable).toContain("403 Forbidden");
expect(d.post).toHaveBeenCalled();
});
});

describe("normalizeLogMessage", () => {
it("keeps the account and product of a path but drops per-request detail", () => {
expect(normalizeLogMessage("status=503, duration_ms=36.0, path=/a/b/c/d.tif, request_id=x1")).toBe(
"status=503, duration_ms=…, path=/a/b/…, request_id=…"
);
});
});
Loading
Loading