diff --git a/workers/events/fixtures/contract-v11.json b/workers/events/fixtures/contract-v11.json index 9926e06..3b0f9b9 100644 --- a/workers/events/fixtures/contract-v11.json +++ b/workers/events/fixtures/contract-v11.json @@ -326,6 +326,8 @@ "workspace": "worktree", "delivery": "auto", "bindToCurrentThread": true, + "failureOwner": "code", + "failureCategory": "code-error", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -1651,6 +1653,10 @@ "durationBucket": "5-15s", "failureClass": "permission", "reviewRequired": true, + "language": "python", + "actor": "agent", + "entry": "cell", + "runtimeSource": "managed", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -1683,6 +1689,12 @@ "durationBucket": "5-15s", "failureClass": "permission", "reviewRequired": true, + "failureOwner": "code", + "failureCategory": "code-error", + "language": "python", + "actor": "agent", + "entry": "cell", + "runtimeSource": "managed", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -1715,6 +1727,12 @@ "durationBucket": "5-15s", "failureClass": "permission", "reviewRequired": true, + "failureOwner": "code", + "failureCategory": "code-error", + "language": "python", + "actor": "agent", + "entry": "cell", + "runtimeSource": "managed", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -1747,6 +1765,12 @@ "durationBucket": "5-15s", "failureClass": "permission", "reviewRequired": true, + "failureOwner": "code", + "failureCategory": "code-error", + "language": "python", + "actor": "agent", + "entry": "cell", + "runtimeSource": "managed", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -1779,6 +1803,45 @@ "durationBucket": "5-15s", "failureClass": "permission", "reviewRequired": true, + "failureOwner": "code", + "failureCategory": "code-error", + "language": "python", + "actor": "agent", + "entry": "cell", + "runtimeSource": "managed", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "provider.commands.observed:fallbacks", + "name": "provider.commands.observed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "modelKey": "unknown", + "commandCategory": "unknown", + "commandCount": 0, + "failedCount": 0, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "provider.commands.observed:representative", + "name": "provider.commands.observed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "modelKey": "gpt-5.6-sol", + "commandCategory": "python", + "commandCount": 3, + "failedCount": 1, + "stoppedCount": 1, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -1843,6 +1906,40 @@ "contractRevision": "11" } }, + { + "case": "scient.operation.failed:attribution-hostile", + "name": "scient.operation.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "operationKind": "compute-run", + "trigger": "other", + "durationBucket": "unknown", + "failureClass": "unknown", + "reviewRequired": false, + "failureOwner": "unknown", + "failureCategory": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "provider.commands.observed:command-hostile", + "name": "provider.commands.observed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "codex", + "modelKey": "gpt-5.6-sol", + "commandCategory": "unknown", + "commandCount": 0, + "failedCount": 0, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, { "case": "scient.operation.completed:capture-v2-retained", "name": "scient.operation.completed", @@ -2071,6 +2168,10 @@ "durationBucket": "5-15s", "failureClass": "permission", "reviewRequired": true, + "language": "python", + "actor": "agent", + "entry": "cell", + "runtimeSource": "managed", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2087,6 +2188,12 @@ "durationBucket": "5-15s", "failureClass": "permission", "reviewRequired": true, + "failureOwner": "code", + "failureCategory": "code-error", + "language": "python", + "actor": "agent", + "entry": "cell", + "runtimeSource": "managed", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2103,6 +2210,12 @@ "durationBucket": "5-15s", "failureClass": "permission", "reviewRequired": true, + "failureOwner": "code", + "failureCategory": "code-error", + "language": "python", + "actor": "agent", + "entry": "cell", + "runtimeSource": "managed", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2119,6 +2232,12 @@ "durationBucket": "5-15s", "failureClass": "permission", "reviewRequired": true, + "failureOwner": "code", + "failureCategory": "code-error", + "language": "python", + "actor": "agent", + "entry": "cell", + "runtimeSource": "managed", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" diff --git a/workers/events/src/attributionPipeline.test.ts b/workers/events/src/attributionPipeline.test.ts new file mode 100644 index 0000000..49eacf5 --- /dev/null +++ b/workers/events/src/attributionPipeline.test.ts @@ -0,0 +1,276 @@ +/// +import { mkdtempSync, rmSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { expect, it } from "vitest"; +import worker from "./index"; +import { testDatabase } from "./sqlite.testSupport"; + +/** + * The questions the failure-attribution data exists to answer, run on the real ledger schema. + * Insights reads the same columns; these are the reference definitions. + */ +export const ATTRIBUTION_QUERIES = { + /** Per Scient capability: finished attempts, failures by owner, and Scient's own failures. */ + operationOutcomes: ` + SELECT + json_extract(properties_json, '$.operationKind') AS operation_kind, + count(*) AS finished, + sum(event_name = 'scient.operation.failed') AS failed, + sum(json_extract(properties_json, '$.failureOwner') = 'scient') AS failed_scient, + sum(json_extract(properties_json, '$.failureOwner') = 'code') AS failed_code, + sum(json_extract(properties_json, '$.failureOwner') = 'environment') AS failed_environment, + sum(json_extract(properties_json, '$.failureOwner') = 'external') AS failed_external, + sum(json_extract(properties_json, '$.failureOwner') = 'unknown') AS failed_unknown + FROM analytics_events + WHERE event_name IN ('scient.operation.completed', 'scient.operation.failed') + GROUP BY operation_kind + ORDER BY operation_kind`, + /** Scient tool calls agents made that failed, by owner. */ + toolFailures: ` + SELECT + json_extract(properties_json, '$.tool') AS tool, + json_extract(properties_json, '$.failureOwner') AS failure_owner, + count(*) AS calls + FROM analytics_events + WHERE event_name = 'mcp.tool.invoked' AND json_extract(properties_json, '$.outcome') = 'error' + GROUP BY tool, failure_owner + ORDER BY tool, failure_owner`, + /** What agents ran in their own terminal instead of, or beyond, Scient's tools. */ + commandsOutsideScient: ` + SELECT + json_extract(properties_json, '$.commandCategory') AS category, + sum(json_extract(properties_json, '$.commandCount')) AS commands, + sum(json_extract(properties_json, '$.failedCount')) AS failed, + count(DISTINCT distinct_id) AS installations + FROM analytics_events + WHERE event_name = 'provider.commands.observed' + GROUP BY category + ORDER BY commands DESC, category`, +} as const; + +// Opt-in cross-repository qualification: build the exact desktop candidate first. +const desktopRoot = process.env.SCIENT_ANALYTICS_DESKTOP_ROOT; +it.skipIf(!desktopRoot)( + "carries failure owners and agent command categories from the desktop producers to the ledger", + async () => { + const root = resolve(desktopRoot!); + const load = (path: string) => import(/* @vite-ignore */ pathToFileURL(join(root, path)).href); + const { createAnalyticsRuntime } = await load("packages/scient-analytics/src/runtime.ts"); + const attribution = await load("apps/server/src/telemetry/FailureAttribution.ts"); + const { classifyCommand } = await load("apps/server/src/telemetry/CommandCategories.ts"); + const fixture = mkdtempSync(join(tmpdir(), "scient-attribution-proof-")); + const store = testDatabase(); + const env = { + ANALYTICS_DB: store.database, + DESKTOP_INGESTION_ENABLED: "true", + ANALYTICS_INGESTION_RATE_LIMITER: { limit: async () => ({ success: true }) }, + }; + const context = { waitUntil: () => undefined } as unknown as ExecutionContext; + const uploads: string[] = []; + const server = createServer((request, response) => { + void (async () => { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const body = Buffer.concat(chunks).toString("utf8"); + uploads.push(body); + const result = await worker.fetch!( + new Request(`http://127.0.0.1${request.url}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Scient-Installation-Token": String( + request.headers["x-scient-installation-token"] ?? "", + ), + }, + body, + }) as Request, + env, + context, + ); + response.writeHead(result.status, { "Content-Type": "application/json" }); + response.end(await result.text()); + })().catch(() => { + response.writeHead(500); + response.end(); + }); + }); + await new Promise((done) => server.listen(0, "127.0.0.1", done)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("No fixture port"); + const runtime = createAnalyticsRuntime({ + enabled: true, + consent: "product", + outboxPath: join(fixture, "outbox.sqlite"), + endpoint: `http://127.0.0.1:${address.port}/v1/events`, + workerUrl: pathToFileURL(join(root, "apps/server/dist/analytics-worker.mjs")), + appVersion: "0.6.8", + buildChannel: "development", + }); + try { + const run = (status: "completed" | "failed", extra: Record = {}) => + runtime.record(`scient.operation.${status}`, { + operationKind: "compute-run", + trigger: "other", + durationMs: 900, + language: "python", + actor: "agent", + entry: "console", + runtimeSource: "managed", + ...extra, + }); + run("completed"); + run("completed"); + // The user's code raised: counted, but not as Scient's failure. + run("failed", { + ...attribution.computeRunAttribution( + { kind: "runtime-reported", errorName: "ZeroDivisionError" }, + true, + ), + }); + run("failed", attribution.computeRunAttribution({ kind: "not-admitted" }, true)); + run( + "failed", + attribution.computeRunAttribution( + { kind: "runtime-reported", errorName: "ModuleNotFoundError" }, + false, + ), + ); + runtime.record("scient.operation.failed", { + operationKind: "latex-build", + trigger: "other", + failureClass: "missing_package", + ...attribution.latexBuildAttribution("missing_package"), + }); + runtime.record("scient.operation.failed", { + operationKind: "compute-artifact", + trigger: "other", + captureSchemaVersion: 2, + captureSource: "interactive", + captureOutcome: "failed", + captureReason: "persistence-failed", + ...attribution.captureRejectionAttribution("persistence-failed"), + }); + runtime.record("mcp.tool.invoked", { + tool: "preview_act", + callerProvider: "codex", + outcome: "error", + errorCode: "invalid_selector", + failureOwner: attribution.toolFailureAttribution("invalid_selector").failureOwner, + }); + runtime.record("mcp.tool.invoked", { + tool: "preview_act", + callerProvider: "codex", + outcome: "error", + errorCode: "no_host", + failureOwner: attribution.toolFailureAttribution("no_host").failureOwner, + }); + // Agent terminal commands, as the turn observer tallies them: category only. + const commands = [ + '/bin/zsh -lc "python /Users/someone/PRIVATE-STUDY/fit.py"', + "python3 -m pip install PRIVATE-PACKAGE", + "stata-mp -b do PRIVATE.do", + "stata-mp -b do PRIVATE-2.do", + ]; + const tallies = new Map(); + for (const command of commands) + for (const category of classifyCommand(command)) + tallies.set(category, (tallies.get(category) ?? 0) + 1); + for (const [commandCategory, commandCount] of tallies) + runtime.record("provider.commands.observed", { + provider: "codex", + model: "gpt-6.1-sol", + commandCategory, + commandCount, + failedCount: commandCategory === "stata" ? 1 : 0, + }); + runtime.record("provider.commands.observed", { + provider: "codex", + model: "gpt-6.1-sol", + commandCategory: "python /Users/someone/PRIVATE-STUDY/fit.py", + commandCount: 1, + failedCount: 0, + }); + + expect(await runtime.flush()).toBe(13); + expect(store.sqlite.prepare("SELECT count(*) AS n FROM analytics_events").get()!.n).toBe(13); + expect(uploads.join("\n")).not.toMatch(/PRIVATE|someone|ZeroDivisionError/u); + + const rows = (sql: string) => + store.sqlite + .prepare(sql) + .all() + .map((row) => ({ ...row })); + expect(rows(ATTRIBUTION_QUERIES.operationOutcomes)).toEqual([ + { + operation_kind: "compute-artifact", + finished: 1, + failed: 1, + failed_scient: 1, + failed_code: 0, + failed_environment: 0, + failed_external: 0, + failed_unknown: 0, + }, + { + operation_kind: "compute-run", + finished: 5, + failed: 3, + failed_scient: 1, + failed_code: 1, + failed_environment: 1, + failed_external: 0, + failed_unknown: 0, + }, + { + operation_kind: "latex-build", + finished: 1, + failed: 1, + failed_scient: 0, + failed_code: 0, + failed_environment: 1, + failed_external: 0, + failed_unknown: 0, + }, + ]); + expect(rows(ATTRIBUTION_QUERIES.toolFailures)).toEqual([ + { tool: "preview_act", failure_owner: "input", calls: 1 }, + { tool: "preview_act", failure_owner: "scient", calls: 1 }, + ]); + expect(rows(ATTRIBUTION_QUERIES.commandsOutsideScient)).toEqual([ + { category: "stata", commands: 2, failed: 1, installations: 1 }, + { category: "python", commands: 1, failed: 0, installations: 1 }, + { category: "python-package-install", commands: 1, failed: 0, installations: 1 }, + // Free text never becomes a category. + { category: "unknown", commands: 1, failed: 0, installations: 1 }, + ]); + const failedRun = JSON.parse( + String( + store.sqlite + .prepare( + "SELECT properties_json FROM analytics_events WHERE event_name = 'scient.operation.failed' AND json_extract(properties_json, '$.failureOwner') = 'code'", + ) + .get()!.properties_json, + ), + ); + expect(failedRun).toMatchObject({ + operationKind: "compute-run", + failureOwner: "code", + failureCategory: "code-error", + language: "python", + actor: "agent", + entry: "console", + runtimeSource: "managed", + contractRevision: "11", + }); + } finally { + await runtime.close(); + server.closeAllConnections(); + await new Promise((done) => server.close(() => done())); + store.close(); + rmSync(fixture, { recursive: true, force: true }); + } + }, +);