From 6d1a92a809178c0798c418e19051c621c30be6f9 Mon Sep 17 00:00:00 2001 From: Vjeran Grozdanic Date: Fri, 9 Oct 2026 15:28:13 +0200 Subject: [PATCH] feat(agent-conversation): Show aggregate metrics Preserve detail stats through pagination and expose costs, call totals, errors, token totals, and trace IDs in conversation output. Refs TET-2959 Refs TET-2951 Co-Authored-By: GPT-5.6 Sol --- .../src/commands/agent-conversation/view.ts | 7 ++- packages/cli/src/lib/api/conversations.ts | 6 +- .../cli/src/lib/formatters/conversation.ts | 63 +++++++++++++++++-- packages/cli/src/types/conversation.ts | 13 ++++ packages/cli/src/types/index.ts | 2 + .../commands/agent-conversation/view.test.ts | 19 +++++- .../cli/test/lib/api/conversations.test.ts | 18 +++++- .../test/lib/formatters/conversation.test.ts | 44 ++++++++++--- 8 files changed, 154 insertions(+), 18 deletions(-) diff --git a/packages/cli/src/commands/agent-conversation/view.ts b/packages/cli/src/commands/agent-conversation/view.ts index 64e7e432d..512db0591 100644 --- a/packages/cli/src/commands/agent-conversation/view.ts +++ b/packages/cli/src/commands/agent-conversation/view.ts @@ -136,7 +136,7 @@ export const viewCommand = buildCommand({ } const org = resolved.org; - const { spans, truncated, title } = await withProgress( + const { spans, stats, truncated, title } = await withProgress( { message: "Fetching conversation spans...", json: flags.json, @@ -144,7 +144,10 @@ export const viewCommand = buildCommand({ () => getConversationSpans(org, conversationId), ); - const result = buildTranscriptResult(conversationId, org, spans, title); + const result = buildTranscriptResult(conversationId, org, spans, { + stats, + title, + }); result.truncated = truncated; yield new CommandOutput(result); }, diff --git a/packages/cli/src/lib/api/conversations.ts b/packages/cli/src/lib/api/conversations.ts index f2b0ebc5f..76f313fdd 100644 --- a/packages/cli/src/lib/api/conversations.ts +++ b/packages/cli/src/lib/api/conversations.ts @@ -20,6 +20,7 @@ import { type AgentConversationSpan, type ConversationListItem, ConversationListItemSchema, + type ConversationStats, } from "../../types/conversation.js"; import { logger } from "../logger.js"; @@ -126,6 +127,7 @@ export async function getConversationSpans( } = {}, ): Promise<{ spans: AgentConversationSpan[]; + stats?: ConversationStats; truncated: boolean; title: string | null; }> { @@ -140,6 +142,7 @@ export async function getConversationSpans( } const spans: AgentConversationSpan[] = []; + let stats: ConversationStats | undefined; let title: string | null = null; let cursor: string | undefined; @@ -156,6 +159,7 @@ export async function getConversationSpans( ); if (page === 0) { + stats = data.stats; title = data.title; } spans.push(...data.spans); @@ -173,5 +177,5 @@ export async function getConversationSpans( ); } - return { spans, truncated, title }; + return { spans, stats, truncated, title }; } diff --git a/packages/cli/src/lib/formatters/conversation.ts b/packages/cli/src/lib/formatters/conversation.ts index 1567c5229..27a1db5ee 100644 --- a/packages/cli/src/lib/formatters/conversation.ts +++ b/packages/cli/src/lib/formatters/conversation.ts @@ -8,6 +8,7 @@ import type { AgentConversationSpan, ConversationListItem, + ConversationStats, } from "../../types/conversation.js"; import { colorTag, @@ -37,10 +38,20 @@ function formatTimestamp(epochSeconds: number): string { return new Date(epochSeconds * 1000).toLocaleString(); } +const USD_FORMAT = new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 6, +}); + +function formatCost(cost: number): string { + return USD_FORMAT.format(cost); +} + /** * Minimum terminal width (columns) to show the full conversation table. * Below this, Tools/Errs/User are dropped so the core ID/Title/Started/ - * Tokens/First Input columns don't truncate aggressively. + * Tokens/Cost/First Input columns don't truncate aggressively. */ const WIDE_TABLE_MIN_TERM_WIDTH = 100; @@ -71,6 +82,12 @@ const TOKENS_COLUMN: Column = { align: "right", }; +const COST_COLUMN: Column = { + header: "Cost", + value: (c) => formatCost(c.totalCost), + align: "right", +}; + const TOOLS_COLUMN: Column = { header: "Tools", value: (c) => String(c.toolCalls), @@ -109,6 +126,7 @@ function selectConversationColumns(): Column[] { TITLE_COLUMN, STARTED_COLUMN, TOKENS_COLUMN, + COST_COLUMN, TOOLS_COLUMN, ERRS_COLUMN, USER_COLUMN, @@ -120,6 +138,7 @@ function selectConversationColumns(): Column[] { TITLE_COLUMN, STARTED_COLUMN, TOKENS_COLUMN, + COST_COLUMN, FIRST_INPUT_COLUMN, ]; } @@ -440,7 +459,13 @@ export type TranscriptResult = { /** Stored conversation title, or null when none has been titled yet. */ title: string | null; turns: ConversationTurn[]; + errors?: number; + llmCalls?: number; + toolCalls?: number; + toolErrors?: number; + totalCost?: number; totalTokens: number; + traceIds: string[]; spanCount: number; projects: string[]; startTimestamp: number; @@ -468,6 +493,21 @@ export function formatTranscriptResult(result: TranscriptResult): string { ["Spans", String(result.spanCount)], ["Tokens", String(result.totalTokens)], ]; + if (result.totalCost !== undefined) { + rows.push(["Cost", formatCost(result.totalCost)]); + } + if (result.llmCalls !== undefined) { + rows.push(["LLM Calls", String(result.llmCalls)]); + } + if (result.toolCalls !== undefined) { + rows.push(["Tool Calls", String(result.toolCalls)]); + } + if (result.errors !== undefined) { + rows.push(["Errors", String(result.errors)]); + } + if (result.toolErrors !== undefined) { + rows.push(["Tool Errors", String(result.toolErrors)]); + } if (result.title) { rows.unshift(["Title", escapeMarkdownCell(result.title)]); } @@ -494,17 +534,30 @@ export function buildTranscriptResult( conversationId: string, org: string, spans: AgentConversationSpan[], - title: string | null = null, + details: { stats?: ConversationStats; title?: string | null } = {}, ): TranscriptResult { + const { stats, title = null } = details; const turns = extractTurns(spans); + const summary = stats + ? { + errors: stats.errors, + llmCalls: stats.llmCalls, + toolCalls: stats.toolCalls, + toolErrors: stats.toolErrors, + totalCost: stats.totalCost, + } + : {}; return { conversationId, org, title, turns, - // Sum tokens per turn (ai_client spans only). Summing every span would - // double-count parent invoke_agent spans that aggregate child usage. - totalTokens: turns.reduce((sum, t) => sum + t.totalTokens, 0), + ...summary, + // Prefer server totals. They cover the whole conversation without + // double-counting parent agent spans. + totalTokens: + stats?.totalTokens ?? turns.reduce((sum, t) => sum + t.totalTokens, 0), + traceIds: [...new Set(spans.map((s) => s.trace))].sort(), spanCount: spans.length, projects: [...new Set(spans.map((s) => s.project))].sort(), startTimestamp: diff --git a/packages/cli/src/types/conversation.ts b/packages/cli/src/types/conversation.ts index 23989ff11..87b83649d 100644 --- a/packages/cli/src/types/conversation.ts +++ b/packages/cli/src/types/conversation.ts @@ -96,6 +96,17 @@ export type AgentConversationSpan = InferOutput< typeof AgentConversationSpanSchema >; +export const ConversationStatsSchema = looseObject({ + errors: number(), + llmCalls: number(), + toolCalls: number(), + toolErrors: number(), + totalCost: number(), + totalTokens: number(), +}); + +export type ConversationStats = InferOutput; + /** * Conversation details envelope returned by * `GET /organizations/{org}/agents/conversations/{conversationId}/`. @@ -109,6 +120,8 @@ export const AgentConversationDetailsSchema = looseObject({ conversationId: string(), title: nullable(string()), spans: array(AgentConversationSpanSchema), + /** Added after the original envelope; optional for older self-hosted servers. */ + stats: optional(ConversationStatsSchema), }); export type AgentConversationDetails = InferOutput< diff --git a/packages/cli/src/types/index.ts b/packages/cli/src/types/index.ts index e803731e3..e046aeccb 100644 --- a/packages/cli/src/types/index.ts +++ b/packages/cli/src/types/index.ts @@ -24,11 +24,13 @@ export type { AgentConversationDetails, AgentConversationSpan, ConversationListItem, + ConversationStats, } from "./conversation.js"; export { AgentConversationDetailsSchema, AgentConversationSpanSchema, ConversationListItemSchema, + ConversationStatsSchema, } from "./conversation.js"; // Dashboard types export type { diff --git a/packages/cli/test/commands/agent-conversation/view.test.ts b/packages/cli/test/commands/agent-conversation/view.test.ts index 7fa0f9c7e..dacb3fe56 100644 --- a/packages/cli/test/commands/agent-conversation/view.test.ts +++ b/packages/cli/test/commands/agent-conversation/view.test.ts @@ -318,6 +318,14 @@ describe("viewCommand.func", () => { test("yields CommandOutput with transcript result (JSON)", async () => { getConversationSpansSpy.mockResolvedValue({ spans: sampleSpans, + stats: { + errors: 1, + llmCalls: 2, + toolCalls: 3, + toolErrors: 1, + totalCost: 0.0042, + totalTokens: 150, + }, truncated: false, title: null, }); @@ -330,7 +338,16 @@ describe("viewCommand.func", () => { const parsed = JSON.parse(output); expect(parsed.conversationId).toBe(CONVERSATION_ID); expect(parsed.org).toBe(ORG); - expect(parsed.spanCount).toBe(2); + expect(parsed).toMatchObject({ + errors: 1, + llmCalls: 2, + toolCalls: 3, + toolErrors: 1, + totalCost: 0.0042, + totalTokens: 150, + traceIds: ["00112233445566778899aabbccddeeff"], + spanCount: 2, + }); expect(parsed.turns).toBeDefined(); expect(Array.isArray(parsed.turns)).toBe(true); }); diff --git a/packages/cli/test/lib/api/conversations.test.ts b/packages/cli/test/lib/api/conversations.test.ts index b46e3bc7f..345869c3d 100644 --- a/packages/cli/test/lib/api/conversations.test.ts +++ b/packages/cli/test/lib/api/conversations.test.ts @@ -19,6 +19,7 @@ import { listConversations, } from "../../../src/lib/api/conversations.js"; import { MAX_PAGINATION_PAGES } from "../../../src/lib/api/infrastructure.js"; +import type { ConversationStats } from "../../../src/types/conversation.js"; import { mockFetch, useTestConfigDir } from "../../helpers.js"; // ============================================================================ @@ -311,11 +312,13 @@ describe("getConversationSpans", () => { function makeEnvelope( spans: ReturnType[], title: string | null = "Test conversation", + stats?: ConversationStats, ) { return { conversationId: CONV_ID, title, spans, + ...(stats ? { stats } : {}), }; } @@ -340,15 +343,26 @@ describe("getConversationSpans", () => { expect(url).toContain("statsPeriod=30d"); }); - test("returns spans from a single page envelope", async () => { + test("returns spans and stats from a single page envelope", async () => { const spans = [makeSpan("span-1-aabb1122")]; - mockOk(makeEnvelope(spans, "Refund flow")); + mockOk( + makeEnvelope(spans, "Refund flow", { + errors: 1, + llmCalls: 1, + toolCalls: 2, + toolErrors: 1, + totalCost: 0.0042, + totalTokens: 100, + }), + ); const result = await getConversationSpans(ORG, CONV_ID); expect(result.spans).toHaveLength(1); expect(result.spans[0].span_id).toBe("span-1-aabb1122"); expect(result.title).toBe("Refund flow"); + expect(result.stats?.totalCost).toBe(0.0042); + expect(result.stats?.totalTokens).toBe(100); expect(result.truncated).toBe(false); }); diff --git a/packages/cli/test/lib/formatters/conversation.test.ts b/packages/cli/test/lib/formatters/conversation.test.ts index 99332c1df..30e1bed40 100644 --- a/packages/cli/test/lib/formatters/conversation.test.ts +++ b/packages/cli/test/lib/formatters/conversation.test.ts @@ -18,6 +18,7 @@ function makeListItem( conversationId: "conv-abc-123", startTimestamp: 1_716_500_000, totalTokens: 500, + totalCost: 0.01, toolCalls: 3, errors: 0, firstInput: "Hello world", @@ -76,6 +77,8 @@ describe("formatConversationTable", () => { expect(result).toContain("test@"); expect(result).toContain("Hello"); expect(result).toContain("500"); + expect(result).toContain("Cost"); + expect(result).toContain("$0.01"); // Wide mode keeps the Tools/Errs/User columns. expect(result).toContain("Tools"); expect(result).toContain("User"); @@ -102,6 +105,7 @@ describe("formatConversationTable", () => { expect(result).toContain("conv-"); expect(result).toContain("Hello"); expect(result).toContain("500"); + expect(result).toContain("Cost"); // Lower-value columns are dropped to avoid aggressive truncation. expect(result).not.toContain("Tools"); expect(result).not.toContain("test@"); @@ -237,18 +241,41 @@ describe("buildTranscriptResult", () => { expect(result.org).toBe("my-org"); expect(result.spanCount).toBe(1); expect(result.totalTokens).toBe(100); + expect(result.traceIds).toEqual(["00112233445566778899aabbccddeeff"]); expect(result.projects).toEqual(["my-project"]); expect(result.startTimestamp).toBe(1_716_500_000); expect(result.endTimestamp).toBe(1_716_500_010); }); + test("uses server conversation summary", () => { + const result = buildTranscriptResult("conv-123", "my-org", [makeSpan()], { + stats: { + errors: 1, + llmCalls: 2, + toolCalls: 3, + toolErrors: 1, + totalCost: 0.0042, + totalTokens: 900, + }, + }); + expect(result).toMatchObject({ + errors: 1, + llmCalls: 2, + toolCalls: 3, + toolErrors: 1, + totalCost: 0.0042, + totalTokens: 900, + }); + const output = formatTranscriptResult(result); + expect(output).toContain("$0.0042"); + expect(output).toContain("LLM Calls"); + expect(output).toContain("Tool Errors"); + }); + test("includes title when provided", () => { - const result = buildTranscriptResult( - "conv-123", - "my-org", - [makeSpan()], - "Refund a duplicate charge", - ); + const result = buildTranscriptResult("conv-123", "my-org", [makeSpan()], { + title: "Refund a duplicate charge", + }); expect(result.title).toBe("Refund a duplicate charge"); }); @@ -278,6 +305,7 @@ describe("formatTranscriptResult", () => { title: null, turns: [], totalTokens: 0, + traceIds: [], spanCount: 0, projects: [], startTimestamp: 0, @@ -295,6 +323,7 @@ describe("formatTranscriptResult", () => { title: "Refund a duplicate charge", turns: [], totalTokens: 0, + traceIds: [], spanCount: 0, projects: [], startTimestamp: 0, @@ -322,7 +351,7 @@ describe("formatTranscriptResult", () => { "conv-123", "my-org", [makeSpan()], - "Refund a duplicate charge", + { title: "Refund a duplicate charge" }, ); const output = formatTranscriptResult(transcript); expect(output).toContain("Agent Conversation: conv-123"); @@ -336,6 +365,7 @@ describe("formatTranscriptResult", () => { title: null, turns: [], totalTokens: 0, + traceIds: [], spanCount: 1, projects: [], startTimestamp: 100,