diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 0bd6f5cf8..fdacad929 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -12,6 +12,7 @@ from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation +from gooddata_eval.core.agentic.dashboard_summary import evaluate_agentic_dashboard_summary from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill @@ -44,6 +45,7 @@ class _LfKw(TypedDict, total=False): "agentic_guardrail", "agentic_conversation", "agentic_kda_skill", + "agentic_dashboard_summary", } ) @@ -228,6 +230,27 @@ def _dispatch_agentic( agent_id=agent_id, **lf_kw, ) + elif kind == "agentic_dashboard_summary": + summary_input = item.summary_input + if summary_input is None: + raise ValueError(f"agentic_dashboard_summary item '{item.id}' is missing required 'summary_input'.") + return evaluate_agentic_dashboard_summary( + host=host, + token=token, + workspace_id=workspace_id, + dashboard_id=summary_input.dashboard_id, + expected_output=eo, + # The fixture's own wording is the prompt under test -- a localized fixture is + # only meaningful if its own phrasing is what reaches the agent. + question=item.question, + # Each widget costs an execution, so a fixture asserting on a handful of charts + # can name them instead of paying for the whole dashboard. Same field and meaning + # the headless /summary endpoint gives it; None summarizes everything. + only_visualizations=summary_input.visualizations, + k=k, + agent_id=agent_id, + **lf_kw, + ) elif kind == "agentic_conversation": fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {} return evaluate_agentic_conversation( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py new file mode 100644 index 000000000..6b5d7b05d --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py @@ -0,0 +1,544 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Agentic dashboard-summary evaluation runner. + +Covers the path a user actually takes: the dashboard's "Summarize" menu item drops them +into the assistant with "Summarize this dashboard" pre-filled, so the summary is produced +by the conversational ``dashboard_summary`` skill, not by the dedicated +``POST /api/v1/ai/workspaces/{ws}/summary`` endpoint the ``dashboard_summary`` test kind +exercises. That endpoint sits behind its own feature flag (``ENABLE_GEN_AI_HEADLESS_SUMMARY``) +and serves API/embedding consumers; the two share an idea and nothing else. + +The skill reads the dashboard from ``userContext.view.dashboard`` and collects only the +widgets that carry a ``result_id`` -- a widget without one is dropped from the summarize +scope entirely, which is why sending a bare dashboard id gets "no dashboard charts were +provided" and sending descriptors without results gets "these visualizations need to be +reloaded". In the browser the ids exist because the client has already rendered the +widgets. Here we do the same thing deliberately: walk the dashboard's layout, execute each +insight, and hand back the result ids that execution produced. + +Scoring is unchanged from the single-shot kind -- ``DashboardSummaryEvaluator`` grades free +text against the fixture's ``must_include``/``must_not_include``/``rubric``, so it does not +care which transport produced the summary. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + +import httpx +from gooddata_sdk import GoodDataSdk + +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.chat.sse_client import ChatClient, ChatError +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.evaluators.summary import DashboardSummaryEvaluator +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + DatasetItem, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, +) +from gooddata_eval.core.timing import PhaseTimings, log_timer + +_DEFAULT_K = 1 + +# What the "Summarize" menu item pre-fills, used when no question is supplied. A fixture's +# own question wins: it is the localized or paraphrased wording under test, and the dispatch +# passes it through for exactly that reason. This is the default for direct callers only. +_DEFAULT_PROMPT = "Summarize this dashboard" + + +@dataclass +class DashboardWidget: + """One insight widget of a dashboard, as the assistant needs to see it.""" + + widget_id: str + title: str + visualization_id: str + # None when the execution failed. Mirrors the browser, where a widget that did not + # render carries no result and is therefore left out of the summarize scope. + result_id: str | None = None + + +def _insight_widgets(content: dict) -> list[DashboardWidget]: + """Every insight widget in a dashboard's layout, in document order. + + Walks the whole document rather than a fixed path: dashboards nest widgets in + sections, and tab-based ones nest those again under ``tabs``, so the depth is not + known ahead of time. Widgets whose insight carries no identifier are skipped -- a + rich-text or unresolved widget has nothing to execute. + """ + widgets: list[DashboardWidget] = [] + seen: set[str] = set() + + def walk(node: Any) -> None: + if isinstance(node, dict): + insight = node.get("insight") + if isinstance(insight, dict): + identifier = insight.get("identifier") + viz_id = identifier.get("id") if isinstance(identifier, dict) else None + if viz_id: + widget_id = str(node.get("localIdentifier") or viz_id) + if widget_id not in seen: + seen.add(widget_id) + widgets.append( + DashboardWidget( + widget_id=widget_id, + title=str(node.get("title") or viz_id), + visualization_id=str(viz_id), + ) + ) + for value in node.values(): + walk(value) + elif isinstance(node, list): + for value in node: + walk(value) + + walk(content) + return widgets + + +def _execute_widget(sdk: GoodDataSdk, workspace_id: str, visualization_id: str) -> str: + """Execute one saved visualization and return the result id its execution produced. + + Reuses the table service's own pivot/non-pivot split so the execution matches what the + client would run for that visualization; only the result id is wanted, not the data. + + ``sdk.tables.for_visualization`` is the public equivalent, but it reads the whole result + into an ExecutionTable and returns that instead of the response -- which discards the one + field wanted here and pays for every row to do it. Two of the three helpers it delegates + to are private, so they are checked up front: ``gooddata-sdk`` is depended on as + ``~=1.74.0`` and a patch release may rename them, which would otherwise surface as an + AttributeError on the first widget of a run rather than as a dependency problem. + """ + import gooddata_sdk.table as table_module # noqa: PLC0415 -- private helpers, imported at use site + + missing = [ + name + for name in ("_vis_is_table", "_get_exec_for_pivot", "get_exec_for_non_pivot") + if not hasattr(table_module, name) + ] + if missing: + raise RuntimeError( + f"gooddata_sdk.table no longer provides {', '.join(missing)}, so a dashboard widget cannot be " + "executed for its result id. The installed gooddata-sdk has moved these helpers -- either pin it " + "back or port _execute_widget onto whatever replaced them." + ) + + visualization = sdk.visualizations.get_visualization(workspace_id, visualization_id) + is_pivot = table_module._vis_is_table(visualization) or visualization.has_bucket_of_type( + table_module.BucketType.ROWS + ) + exec_def = ( + table_module._get_exec_for_pivot(visualization) + if is_pivot + else table_module.get_exec_for_non_pivot(visualization) + ) + return sdk.compute.for_exec_def(workspace_id, exec_def).result_id + + +def _fetch_dashboard(host: str, token: str, workspace_id: str, dashboard_id: str) -> tuple[str | None, dict]: + """The dashboard's title and layout document. + + Read over plain HTTP rather than through ``entities_api``: the generated client + validates the entity's timestamp fields with a regex and raises + ``TypeError: expected string or bytes-like object, got 'datetime.datetime'`` on the + response, so the typed accessor cannot read a dashboard at all today. + """ + url = f"{host.rstrip('/')}/api/v1/entities/workspaces/{workspace_id}/analyticalDashboards/{dashboard_id}" + resp = httpx.get(url, headers={"Authorization": f"Bearer {token}"}, timeout=120.0) + resp.raise_for_status() + attributes = resp.json()["data"]["attributes"] + return attributes.get("title"), attributes.get("content") or {} + + +def build_dashboard_user_context( + sdk: GoodDataSdk, + host: str, + token: str, + workspace_id: str, + dashboard_id: str, + *, + only_visualizations: list[str] | None = None, + max_widgets: int | None = None, +) -> tuple[dict, list[DashboardWidget]]: + """Assemble the ``userContext`` for one dashboard, executing its widgets to get result ids. + + Returns the context and the widgets it describes, each carrying the result id its + execution produced or None if that execution failed. A failed widget is still returned + (reporting needs to know the summary was partial) but is left out of the context, since + the skill would drop it anyway. + + A dashboard of thirty widgets costs thirty executions per item, so both arguments exist + to bound that -- at the price of summarizing less than the user would see. + ``only_visualizations`` keeps just the named visualizations, which is what a fixture sets + (via ``summary_input.visualizations``, the same field and meaning the headless endpoint + gives it) because naming the widgets a rubric asserts on stays stable as the dashboard + grows. ``max_widgets`` truncates in layout order and is a blunt cap for direct callers. + """ + title, content = _fetch_dashboard(host, token, workspace_id, dashboard_id) + + widgets = _insight_widgets(content) + if only_visualizations is not None: + wanted = set(only_visualizations) + widgets = [w for w in widgets if w.visualization_id in wanted or w.widget_id in wanted] + if max_widgets is not None: + widgets = widgets[:max_widgets] + + for widget in widgets: + try: + widget.result_id = _execute_widget(sdk, workspace_id, widget.visualization_id) + except Exception as exc: # noqa: BLE001, PERF203 -- one dead widget must not lose the whole summary + log_timer(f"[dashboard_summary] widget '{widget.widget_id}' failed to execute: {exc}") + + context = { + "view": { + "dashboard": { + "id": dashboard_id, + "title": title, + "widgets": [ + { + "widgetId": w.widget_id, + "title": w.title, + "widgetType": "insight", + "visualizationId": w.visualization_id, + "resultId": w.result_id, + } + for w in widgets + if w.result_id is not None + ], + } + } + } + return context, widgets + + +@dataclass +class DashboardSummaryRunResult: + """Outcome of one K-run conversation for a dashboard summary.""" + + conversation_id: str + actual_output: str + evaluation: ItemEvaluation + widgets_total: int + widgets_executed: int + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None + timings: PhaseTimings = field(default_factory=PhaseTimings) + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) + # Set when the chat call itself failed. Such a run has no summary to grade, so it is + # recorded rather than raised -- raising would discard the K-runs already completed. + chat_error: str | None = None + + @property + def passed(self) -> bool: + return self.chat_error is None and self.evaluation.passed + + +@dataclass +class AgenticDashboardSummarySummary: + """Aggregated outcome of K runs for one dashboard-summary item.""" + + run_results: list[DashboardSummaryRunResult] + pass_at_k: bool + pass_power_k: bool + best: DashboardSummaryRunResult + + +def _failed_run( + conversation_id: str, message: str, widgets: list[DashboardWidget], agent_s: float +) -> DashboardSummaryRunResult: + """A run that never produced a summary, recorded so the completed runs survive.""" + return DashboardSummaryRunResult( + conversation_id=conversation_id, + actual_output="", + evaluation=ItemEvaluation(passed=False, rank_key=(-1, 0.0), detail={}, error=message), + widgets_total=len(widgets), + widgets_executed=sum(1 for w in widgets if w.result_id is not None), + timings=PhaseTimings(agent_s=agent_s), + chat_error=message, + ) + + +def _run_single_dashboard_summary( + client: ChatClient, + evaluator: DashboardSummaryEvaluator, + conversation_id: str, + item: DatasetItem, + user_context: dict, + widgets: list[DashboardWidget], + prompt: str, +) -> DashboardSummaryRunResult: + widgets_total = len(widgets) + widgets_executed = sum(1 for w in widgets if w.result_id is not None) + + agent_started = time.monotonic() + try: + chat_result = client.send_message(conversation_id, prompt, user_context=user_context) + except ChatError as exc: + return _failed_run(conversation_id, f"chat failed: {exc}", widgets, time.monotonic() - agent_started) + agent_elapsed = time.monotonic() - agent_started + + judge_started = time.monotonic() + evaluation = evaluator.evaluate(item, chat_result) + judge_elapsed = time.monotonic() - judge_started + log_timer( + f"[timer] dashboard_summary {conversation_id} agent {agent_elapsed:.2f}s, judge {judge_elapsed:.2f}s " + f"({widgets_executed}/{widgets_total} widgets executed)" + ) + + return DashboardSummaryRunResult( + conversation_id=conversation_id, + actual_output=str(evaluation.detail.get("actual_output", "")), + evaluation=evaluation, + widgets_total=widgets_total, + widgets_executed=widgets_executed, + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, + timings=PhaseTimings(agent_s=agent_elapsed, judge_s=judge_elapsed), + tool_call_events=list(chat_result.tool_call_events or []), + reasoning_step_events=list(chat_result.reasoning_step_events or []), + ) + + +def run_agentic_dashboard_summary( + host: str, + token: str, + workspace_id: str, + dashboard_id: str, + expected_output: Any, + question: str = _DEFAULT_PROMPT, + k: int = _DEFAULT_K, + max_iterations: int = 1, # noqa: ARG001 -- single-turn kind; accepted so the runner can dispatch uniformly + initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, + only_visualizations: list[str] | None = None, + max_widgets: int | None = None, + dataset_item_id: str = "", + dataset_name: str = "dashboard_summary", +) -> AgenticDashboardSummarySummary: + """Run the agentic dashboard-summary evaluation K times and return a summary. + + The widgets are executed ONCE and their result ids reused across all K runs: they + identify cached executions, so re-running them per K would multiply the most expensive + part of the item without changing what the assistant sees. + """ + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) + sdk = GoodDataSdk.create(host, token) + evaluator = DashboardSummaryEvaluator() + item = DatasetItem( + id=dataset_item_id or dashboard_id, + dataset_name=dataset_name, + test_kind="agentic_dashboard_summary", + question=question, + expected_output=expected_output, + ) + run_results: list[DashboardSummaryRunResult] = [] + + try: + context_started = time.monotonic() + user_context, widgets = build_dashboard_user_context( + sdk, + host, + token, + workspace_id, + dashboard_id, + only_visualizations=only_visualizations, + max_widgets=max_widgets, + ) + log_timer( + f"[timer] dashboard_summary {dashboard_id} context built in " + f"{time.monotonic() - context_started:.2f}s ({len(widgets)} widgets)" + ) + + conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() + try: + run_results.append( + _run_single_dashboard_summary(client, evaluator, conv_id_0, item, user_context, widgets, question) + ) + finally: + if initial_conversation_id is None: + client.delete_conversation(conv_id_0) + + for _ in range(1, k): + try: + conv_id = client.create_conversation() + except Exception as exc: # noqa: BLE001 -- a lost conversation must not discard completed runs + # Same contract as the ChatError path below: this line used to sit outside any + # handler, so a transient failure here on run 2 of 3 threw away run 1. + run_results.append(_failed_run("", f"conversation creation failed: {exc}", widgets, 0.0)) + continue + try: + run_results.append( + _run_single_dashboard_summary(client, evaluator, conv_id, item, user_context, widgets, question) + ) + finally: + client.delete_conversation(conv_id) + finally: + client.close() + + scored = [r for r in run_results if r.chat_error is None] + pass_at_k = any(r.passed for r in scored) + pass_power_k = len(scored) == len(run_results) and bool(scored) and all(r.passed for r in scored) + best = max(scored or run_results, key=lambda r: r.evaluation.rank_key) + return AgenticDashboardSummarySummary( + run_results=run_results, + pass_at_k=pass_at_k, + pass_power_k=pass_power_k, + best=best, + ) + + +def _detail(summary: AgenticDashboardSummarySummary) -> dict: + best = summary.best + detail = dict(best.evaluation.detail) + # How much of the dashboard the summary actually covered. A summary graded against a + # rubric that mentions a widget which never executed fails for a reason that has + # nothing to do with the agent, so the ratio has to be visible in the report. + detail["widgets_total"] = best.widgets_total + detail["widgets_executed"] = best.widgets_executed + if best.chat_error is not None: + detail["chat_error"] = best.chat_error + detail["latency_breakdown"] = build_latency_breakdown(best.tool_call_events, best.reasoning_step_events) + return detail + + +class DashboardSummaryAssertionError(AgenticAssertionError): + """Raised when an agentic dashboard-summary evaluation fails.""" + + +def evaluate_agentic_dashboard_summary( + host: str, + token: str, + workspace_id: str, + dashboard_id: str, + expected_output: Any, + question: str = _DEFAULT_PROMPT, + k: int = _DEFAULT_K, + initial_conversation_id: str | None = None, + agent_id: str | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "dashboard_summary", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + only_visualizations: list[str] | None = None, + max_widgets: int | None = None, +) -> AgenticEvalOutcome: + """Run the evaluation, log to Langfuse, and raise DashboardSummaryAssertionError on failure.""" + langfuse, window_start = open_trace_window(langfuse) + summary = run_agentic_dashboard_summary( + host=host, + token=token, + workspace_id=workspace_id, + dashboard_id=dashboard_id, + expected_output=expected_output, + question=question, + k=k, + initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + only_visualizations=only_visualizations, + max_widgets=max_widgets, + dataset_item_id=dataset_item_id, + dataset_name=dataset_name, + ) + + if langfuse is not None and dataset_item_id: + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + for run_idx, run in enumerate(summary.run_results): + if run.chat_error is not None: + # The chat never produced a summary, so there is no verdict: writing + # 0.0 would publish a content failure the judge never assessed. + continue + pt = ctx.trace(run.conversation_id) + with ctx.observe(pt, run_idx) as tid: + ctx.score( + tid, name="dashboard_summary_pass", value=float(run.evaluation.passed), data_type="BOOLEAN" + ) + ctx.quality( + tid, + strict_checks={"dashboard_summary_pass": run.evaluation.passed}, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + # A run whose chat failed is skipped above, so polling for its trace would only + # spend the item's shared retry budget on scores that never get written. + conversation_ids=[r.conversation_id for r in summary.run_results if r.chat_error is None], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, + # The question this run answered, so a score is readable without resolving the + # conversation back to its item. + item_input=question, + ) + + best = summary.best + detail = _detail(summary) + timings = PhaseTimings() + for run in summary.run_results: + timings = timings + run.timings + runs_passed = sum(1 for r in summary.run_results if r.passed) + + if not summary.pass_at_k: + error = DashboardSummaryAssertionError( + f"Dashboard summary failed for '{dashboard_id}' " + f"({best.widgets_executed}/{best.widgets_total} widgets executed): " + f"{best.evaluation.error or 'criteria not satisfied'}" + ) + error.reasoning_steps = best.reasoning_steps + error.conversation_id = best.conversation_id + error.response_id = best.response_id + error.detail = detail + error.timings = timings + error.runs_passed = runs_passed + error.runs_effective = len(summary.run_results) + raise error + + return AgenticEvalOutcome( + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + timings=timings, + runs_passed=runs_passed, + runs_effective=len(summary.run_results), + ) diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py b/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py new file mode 100644 index 000000000..431ac50bd --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py @@ -0,0 +1,367 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +from unittest.mock import MagicMock, patch + +import gooddata_sdk +import gooddata_sdk.table as table_module +import pytest +from gooddata_eval.core.agentic.dashboard_summary import ( + DashboardSummaryAssertionError, + DashboardWidget, + _execute_widget, + _insight_widgets, + build_dashboard_user_context, + evaluate_agentic_dashboard_summary, + run_agentic_dashboard_summary, +) +from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.models import ChatResult + +_MODULE = "gooddata_eval.core.agentic.dashboard_summary" + + +def _widget(local_id: str, viz_id: str, title: str = "A chart") -> dict: + return { + "localIdentifier": local_id, + "title": title, + "insight": {"identifier": {"id": viz_id, "type": "visualizationObject"}}, + } + + +def _summary_result(text: str = "## Executive Summary\n- Approvals trended up.") -> ChatResult: + return ChatResult.model_validate({"textResponse": text, "toolCallEvents": [], "reasoningSteps": []}) + + +# ── layout walking ────────────────────────────────────────────────────────── + + +def test_insight_widgets_walks_nested_sections_and_tabs(): + """Widget depth is not fixed: sections nest widgets, and a tab-based dashboard nests + those again, so the walk cannot assume a path.""" + content = { + "version": "3", + "tabs": [ + {"localIdentifier": "tab1", "sections": [{"items": [_widget("w1", "viz-1")]}]}, + {"localIdentifier": "tab2", "sections": [{"items": [_widget("w2", "viz-2"), _widget("w3", "viz-3")]}]}, + ], + } + assert [w.widget_id for w in _insight_widgets(content)] == ["w1", "w2", "w3"] + assert [w.visualization_id for w in _insight_widgets(content)] == ["viz-1", "viz-2", "viz-3"] + + +def test_insight_widgets_skips_widgets_with_nothing_to_execute(): + """Rich text and unresolved widgets carry no insight identifier, so there is nothing to + execute and nothing the summarize scope could use.""" + content = { + "sections": [ + {"items": [_widget("w1", "viz-1"), {"localIdentifier": "rt", "richText": {"content": "hello"}}]}, + {"items": [{"localIdentifier": "broken", "insight": {}}]}, + ] + } + assert [w.widget_id for w in _insight_widgets(content)] == ["w1"] + + +def test_insight_widgets_does_not_repeat_a_widget_reached_twice(): + """The walk descends into every value, so a layout that references one widget from two + places must still yield it once -- a duplicate would be executed twice.""" + shared = _widget("w1", "viz-1") + content = {"sections": [{"items": [shared]}], "alsoHere": {"items": [shared]}} + assert [w.widget_id for w in _insight_widgets(content)] == ["w1"] + + +def test_insight_widget_falls_back_to_the_visualization_id_when_unnamed(): + content = {"items": [{"insight": {"identifier": {"id": "viz-1"}}}]} + (widget,) = _insight_widgets(content) + assert widget.widget_id == "viz-1" + assert widget.title == "viz-1" + + +# ── context building ──────────────────────────────────────────────────────── + + +def _patched_dashboard(widgets: list[dict], title: str = "Sales"): + return patch(f"{_MODULE}._fetch_dashboard", return_value=(title, {"sections": [{"items": widgets}]})) + + +def test_build_user_context_carries_a_result_id_per_widget(): + sdk = MagicMock() + with ( + _patched_dashboard([_widget("w1", "viz-1"), _widget("w2", "viz-2")]), + patch(f"{_MODULE}._execute_widget", side_effect=["res-1", "res-2"]), + ): + context, widgets = build_dashboard_user_context(sdk, "http://h", "tok", "ws1", "dash-1") + + dashboard = context["view"]["dashboard"] + assert dashboard["id"] == "dash-1" + assert dashboard["title"] == "Sales" + assert [w["resultId"] for w in dashboard["widgets"]] == ["res-1", "res-2"] + assert [w["widgetType"] for w in dashboard["widgets"]] == ["insight", "insight"] + assert [w.result_id for w in widgets] == ["res-1", "res-2"] + + +def test_a_widget_that_fails_to_execute_is_reported_but_left_out_of_the_context(): + """The skill drops any widget without a result_id, so sending one would be a silent + no-op -- but the report still has to show the summary covered less than the dashboard.""" + sdk = MagicMock() + with ( + _patched_dashboard([_widget("w1", "viz-1"), _widget("w2", "viz-2")]), + patch(f"{_MODULE}._execute_widget", side_effect=["res-1", RuntimeError("execution blew up")]), + ): + context, widgets = build_dashboard_user_context(sdk, "http://h", "tok", "ws1", "dash-1") + + assert [w["widgetId"] for w in context["view"]["dashboard"]["widgets"]] == ["w1"] + assert len(widgets) == 2 + assert widgets[1].result_id is None + + +def test_max_widgets_caps_the_executions(): + """A thirty-widget dashboard costs thirty executions per item; the cap bounds that.""" + sdk = MagicMock() + layout = [_widget(f"w{i}", f"viz-{i}") for i in range(5)] + with ( + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res") as execute, + ): + _, widgets = build_dashboard_user_context(sdk, "http://h", "tok", "ws1", "dash-1", max_widgets=2) + + assert len(widgets) == 2 + assert execute.call_count == 2 + + +# ── runs ──────────────────────────────────────────────────────────────────── + + +def _run(chat_side_effect, *, k=1, passed=True, widgets=2): + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, k + 2)] + client.send_message.side_effect = chat_side_effect + evaluation = ItemEvaluation(passed=passed, rank_key=(int(passed), 1.0), detail={"actual_output": "text"}) + evaluator = MagicMock() + evaluator.evaluate.return_value = evaluation + layout = [_widget(f"w{i}", f"viz-{i}") for i in range(widgets)] + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res"), + ): + return run_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + k=k, + ) + + +def test_a_passing_run_records_how_much_of_the_dashboard_it_covered(): + summary = _run([_summary_result()]) + assert summary.pass_at_k is True + assert summary.best.widgets_total == 2 + assert summary.best.widgets_executed == 2 + + +def test_widgets_are_executed_once_and_reused_across_k_runs(): + """Result ids identify cached executions, so re-running them per K would multiply the + most expensive part of the item without changing what the assistant sees.""" + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2", "conv-3"] + client.send_message.return_value = _summary_result() + evaluator = MagicMock() + evaluator.evaluate.return_value = ItemEvaluation(passed=True, rank_key=(1, 1.0), detail={"actual_output": "t"}) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard([_widget("w1", "viz-1")]), + patch(f"{_MODULE}._execute_widget", return_value="res") as execute, + ): + summary = run_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + k=3, + ) + + assert len(summary.run_results) == 3 + assert execute.call_count == 1 + + +def test_a_chat_error_is_recorded_on_the_run_rather_than_raised(): + """Raising would discard the K-runs already completed and leave the item with no + verdict at all -- the same failure mode fixed for the other agentic kinds.""" + summary = _run([ChatError("gen-ai fell over")]) + + assert summary.pass_at_k is False + run = summary.run_results[0] + assert run.chat_error is not None + assert "gen-ai fell over" in run.chat_error + assert run.passed is False + assert run.evaluation.error is not None + + +def test_a_chat_error_on_a_later_run_does_not_discard_the_earlier_one(): + summary = _run([_summary_result(), ChatError("boom")], k=2) + + assert len(summary.run_results) == 2 + assert summary.run_results[0].chat_error is None + assert summary.run_results[1].chat_error is not None + assert summary.pass_at_k is True # run 0 still counts + # The chat-error run is unscored, so it cannot certify that every run passed. + assert summary.pass_power_k is False + + +def test_the_best_run_is_never_a_chat_error_when_a_graded_run_exists(): + summary = _run([ChatError("boom"), _summary_result()], k=2) + assert summary.best.chat_error is None + + +# ── evaluate_* wrapper ────────────────────────────────────────────────────── + + +def _evaluate(passed: bool): + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = _summary_result() + evaluator = MagicMock() + evaluator.evaluate.return_value = ItemEvaluation( + passed=passed, rank_key=(int(passed), 1.0), detail={"actual_output": "text", "include_0": passed} + ) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard([_widget("w1", "viz-1"), _widget("w2", "viz-2")]), + patch(f"{_MODULE}._execute_widget", side_effect=["res-1", RuntimeError("dead")]), + ): + return evaluate_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + ) + + +def test_detail_reports_coverage_and_the_latency_breakdown(): + outcome = _evaluate(passed=True) + # A rubric naming a widget that never executed fails for a reason that is not the + # agent's, so the ratio has to be visible without opening the trace. + assert outcome.detail["widgets_total"] == 2 + assert outcome.detail["widgets_executed"] == 1 + assert "latency_breakdown" in outcome.detail + assert outcome.detail["include_0"] is True + assert outcome.conversation_id == "conv-1" + + +def test_a_failing_item_raises_with_the_same_detail_attached(): + with pytest.raises(DashboardSummaryAssertionError) as exc_info: + _evaluate(passed=False) + + error = exc_info.value + assert "1/2 widgets executed" in str(error) + assert error.detail["widgets_executed"] == 1 + assert error.runs_passed == 0 + assert error.runs_effective == 1 + assert error.conversation_id == "conv-1" + + +def test_dashboard_widget_defaults_to_no_result(): + assert DashboardWidget(widget_id="w", title="t", visualization_id="v").result_id is None + + +def test_only_visualizations_keeps_just_the_named_charts(): + """A fixture asserting on a handful of charts should not pay to execute thirty. Names + are matched against the visualization id and the widget id, since a fixture author + reading a dashboard's AAC sees both.""" + sdk = MagicMock() + layout = [_widget("w0", "viz-0"), _widget("w1", "viz-1"), _widget("w2", "viz-2")] + with ( + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res") as execute, + ): + _, widgets = build_dashboard_user_context( + sdk, "http://h", "tok", "ws1", "dash-1", only_visualizations=["viz-1", "w2"] + ) + + assert [w.widget_id for w in widgets] == ["w1", "w2"] + assert execute.call_count == 2 + + +def test_only_visualizations_and_max_widgets_compose(): + sdk = MagicMock() + layout = [_widget(f"w{i}", f"viz-{i}") for i in range(4)] + with ( + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res"), + ): + _, widgets = build_dashboard_user_context( + sdk, + "http://h", + "tok", + "ws1", + "dash-1", + only_visualizations=["viz-1", "viz-2", "viz-3"], + max_widgets=2, + ) + + assert [w.widget_id for w in widgets] == ["w1", "w2"] + + +def test_a_failed_conversation_creation_does_not_discard_completed_runs(): + """create_conversation sat outside any handler, so a transient failure on run 2 of 3 + threw away run 1 -- the same failure mode the ChatError path already guards.""" + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", RuntimeError("no conversation for you"), "conv-3"] + client.send_message.return_value = _summary_result() + evaluator = MagicMock() + evaluator.evaluate.return_value = ItemEvaluation(passed=True, rank_key=(1, 1.0), detail={"actual_output": "t"}) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard([_widget("w1", "viz-1")]), + patch(f"{_MODULE}._execute_widget", return_value="res"), + ): + summary = run_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + k=3, + ) + + assert len(summary.run_results) == 3 + assert [r.chat_error is None for r in summary.run_results] == [True, False, True] + assert "conversation creation failed" in summary.run_results[1].chat_error + assert summary.pass_at_k is True + # The lost run has no conversation to score, so it must not certify an all-passed item. + assert summary.pass_power_k is False + + +def test_execute_widget_fails_loudly_when_the_sdk_moves_its_private_helpers(): + """gooddata-sdk is depended on as ~=1.74.0, so a patch release may rename the private + helpers this borrows. Without the check that surfaces as an AttributeError on the first + widget of a run rather than as the dependency problem it is.""" + stripped = MagicMock(spec=[]) # a gooddata_sdk.table exposing none of the three helpers + # Both bindings: `import a.b as c` reads the parent package's attribute, while + # sys.modules is what keeps a re-import from restoring the real one. + with ( + patch.dict("sys.modules", {"gooddata_sdk.table": stripped}), + patch.object(gooddata_sdk, "table", stripped), + pytest.raises(RuntimeError, match="no longer provides"), + ): + _execute_widget(MagicMock(), "ws1", "viz-1") + + +def test_execute_widget_guard_passes_against_the_installed_sdk(): + """The guard must not be a permanent tripwire: the helpers exist in the pinned version, + so a real call gets past it and fails (if at all) on the network, not on the check.""" + for name in ("_vis_is_table", "_get_exec_for_pivot", "get_exec_for_non_pivot"): + assert hasattr(table_module, name), f"gooddata_sdk.table.{name} is gone -- update _execute_widget" diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 77086d101..cf2eecdcb 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -85,8 +85,17 @@ def test_dispatch_agentic_omits_agent_id_by_default(): ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), ("agentic_kda_skill", {"Measure": {"type": "metric", "id": "revenue"}}, "evaluate_agentic_kda_skill"), ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), + ( + "agentic_dashboard_summary", + {"must_include": ["States the overall trend."]}, + "evaluate_agentic_dashboard_summary", + ), ] +# The one kind whose dispatch needs more than question/expected_output: the dashboard to +# summarize is named in summary_input, exactly as for the single-shot dashboard_summary. +_SUMMARY_INPUT_KINDS = {"agentic_dashboard_summary"} + def test_all_agentic_kind_cases_covers_every_registered_kind(): """Guards the two parametrized tests below against silently going stale: a kind added @@ -104,6 +113,7 @@ def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_ test_kind=kind, question="q", expected_output=expected_output, + summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, ) with patch(f"gooddata_eval.cli.agentic_runner.{target}") as mock_eval: _dispatch_agentic( @@ -209,6 +219,7 @@ def test_dispatch_agentic_returns_a_real_outcome_for_every_kind(kind, expected_o test_kind=kind, question="q", expected_output=expected_output, + summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, ) canned = AgenticEvalOutcome(reasoning_steps=["x"], conversation_id="c1", response_id="r1", detail={"k": "v"}) with patch(f"gooddata_eval.cli.agentic_runner.{target}", return_value=canned) as mock_eval: diff --git a/packages/gooddata-eval/tests/test_trace_linker.py b/packages/gooddata-eval/tests/test_trace_linker.py index f490effd6..20715e413 100644 --- a/packages/gooddata-eval/tests/test_trace_linker.py +++ b/packages/gooddata-eval/tests/test_trace_linker.py @@ -118,6 +118,7 @@ def test_run_trace_link_inline_runs_the_task_on_the_calling_thread(): ("visualization", "evaluate_agentic_visualization"), ("kda_skill", "evaluate_agentic_kda_skill"), ("conversation", "evaluate_agentic_conversation"), + ("dashboard_summary", "evaluate_agentic_dashboard_summary"), ]