diff --git a/packages/codev-upload/src/stats.ts b/packages/codev-upload/src/stats.ts index d3370c3b6..8cde1c47e 100644 --- a/packages/codev-upload/src/stats.ts +++ b/packages/codev-upload/src/stats.ts @@ -17,6 +17,8 @@ export interface StatsMessage { role?: string modelID?: string providerID?: string + agent?: string + mode?: string cost?: number tokens?: { input?: number @@ -89,6 +91,15 @@ export interface SessionStats { user_turns: number subagents: number files_changed: number + // Assistant messages per agent (build, plan, ...) in the top-level session. + agents: Record + // Skill uses by name: skill tool calls and reads of a skill's SKILL.md, + // subagents included. + skills: Record + // Skills whose SKILL.md the session wrote, by name. + skills_created: Record + // Distinct files read or modified, by lowercased extension (".ts"). + file_extensions: Record totals: Usage buckets: Bucket[] } @@ -132,6 +143,12 @@ export function buildSessionStats(input: StatsInput): SessionStats | null { user_turns: input.messages.filter(isUserPrompt).length, subagents: input.descendants.length, files_changed: files.size, + agents: tally( + input.messages.flatMap((message) => + message.info.role === "assistant" ? [message.info.agent || message.info.mode || ""].filter(Boolean) : [], + ), + ), + ...fileBreakdowns([...input.messages, ...input.descendants.flat()]), totals, buckets: [...buckets.values()].sort((a, b) => a.hour.localeCompare(b.hour) || a.model.localeCompare(b.model)), } @@ -240,6 +257,57 @@ function toolEdit(part: FinishedToolPart): { added: number; deleted: number; fil return undefined } +// Skill and file breakdowns from every settled tool call. Log-analytics +// parity (ai-hub-worker's skills and file_paths enrichers): a read of +// /SKILL.md is a skill use, a write to it creates the skill, and file +// extensions cover files read as well as modified. +function fileBreakdowns(messages: StatsMessage[]) { + const skills: string[] = [] + const created = new Set() + const touched = new Set() + for (const part of messages.flatMap((message) => message.parts.filter(isFinishedToolPart))) { + const input = part.state.input ?? {} + if (part.tool === "skill") { + const name = textValue(input.name) + if (name) skills.push(name) + continue + } + const paths = toolPaths(part) + for (const path of paths) touched.add(path) + for (const name of paths.flatMap((path) => SKILL_FILE.exec(path)?.slice(1, 2) ?? [])) { + if (part.tool === "read") skills.push(name) + else if (part.state.status === "completed") created.add(name) + } + } + return { + skills: tally(skills), + skills_created: tally([...created]), + file_extensions: tally([...touched].map(extensionOf).filter(Boolean)), + } +} + +const SKILL_FILE = /[/\\]([^/\\]+)[/\\]SKILL\.md$/ +const FILE_TOOLS = new Set(["read", "edit", "write"]) + +function toolPaths(part: FinishedToolPart): string[] { + const input = part.state.input ?? {} + if (FILE_TOOLS.has(part.tool)) return [textValue(input.filePath)].filter(Boolean) + if (part.tool !== "apply_patch" || !Array.isArray(part.state.metadata?.files)) return [] + return (part.state.metadata.files as { filePath?: unknown }[]).map((file) => textValue(file.filePath)).filter(Boolean) +} + +function extensionOf(path: string): string { + const name = path.split(/[/\\]/).pop() ?? "" + const dot = name.lastIndexOf(".") + return dot > 0 ? name.slice(dot).toLowerCase() : "" +} + +function tally(values: string[]): Record { + const counts: Record = {} + for (const value of values) counts[value] = (counts[value] ?? 0) + 1 + return counts +} + function countDiffLines(diff: string): { added: number; deleted: number } { const lines = diff.split("\n") return { diff --git a/packages/codev-upload/test/stats.test.ts b/packages/codev-upload/test/stats.test.ts index 91b09f071..c079ea544 100644 --- a/packages/codev-upload/test/stats.test.ts +++ b/packages/codev-upload/test/stats.test.ts @@ -135,6 +135,37 @@ describe("buildSessionStats", () => { expect(stats?.files_changed).toBe(2) }) + test("breaks sessions down by agent, skill, created skill, and file extension", () => { + const stats = buildSessionStats({ + id: "ses1", + origin: "local", + project: "proj", + messages: [ + assistant(T0, { agent: "build" }, [ + tool("skill", "completed", { input: { name: "pdf" } }), + tool("read", "completed", { input: { filePath: "/home/u/.codev/skills/xlsx/SKILL.md" } }), + tool("read", "completed", { input: { filePath: "/p/src/App.TSX" } }), + tool("edit", "completed", { input: { filePath: "/p/src/app.tsx" } }), + tool("write", "completed", { input: { filePath: "/p/.codev/skills/deploy/SKILL.md" } }), + // A denied write creates nothing, but the file was still touched. + tool("write", "error", { input: { filePath: "/p/.codev/skills/broken/SKILL.md" } }), + tool("apply_patch", "completed", { + metadata: { files: [{ filePath: "/p/README" }, { filePath: "/p/a.py" }] }, + }), + ]), + assistant(T0 + 1000, { agent: "plan" }), + assistant(T0 + 2000, { agent: "build" }), + ], + descendants: [[assistant(T0 + 1500, { agent: "explore" }, [tool("skill", "error", { input: { name: "pdf" } })])]], + }) + // Subagent turns belong to the subagent, not the session's mode. + expect(stats?.agents).toEqual({ build: 2, plan: 1 }) + expect(stats?.skills).toEqual({ pdf: 2, xlsx: 1 }) + expect(stats?.skills_created).toEqual({ deploy: 1 }) + // Distinct files touched per extension, case-folded; extensionless files are skipped. + expect(stats?.file_extensions).toEqual({ ".md": 3, ".tsx": 2, ".py": 1 }) + }) + test("a session without messages yields nothing", () => { expect(buildSessionStats({ id: "s", origin: "local", project: "p", messages: [], descendants: [] })).toBeNull() })