From 4e3ba701bda7bcdcb1858fc2575ac3627f19e55f Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Wed, 9 Sep 2026 23:52:16 +0200 Subject: [PATCH 1/3] feat(gooddata-eval): evaluate the dashboard summary users actually get The dashboard_summary kind drives POST /api/v1/ai/workspaces/{ws}/summary. Nothing in the product reaches that endpoint: the dashboard's "Summarize" menu item drops the user into the assistant with "Summarize this dashboard" pre-filled, so real summaries come from the conversational dashboard_summary skill. The endpoint is a separate capability for API/embedding consumers, behind its own feature flag (ENABLE_GEN_AI_HEADLESS_SUMMARY), enforced in afm-exec-api before gen-ai is reached -- which is why all 36 dashboard_summary results ever recorded are 400s and not one summary has been graded. Adds agentic_dashboard_summary, covering the path users take. Getting the skill to engage needs more than a dashboard id. It reads userContext.view.dashboard and keeps only widgets carrying a result_id -- one without is dropped from the summarize scope, so an id alone answers "no dashboard charts were provided" and bare descriptors answer "these visualizations need to be reloaded". The browser has those ids because it rendered the widgets, so the runner does the same deliberately: walk the layout, execute each insight, send the result ids execution produced. Verified live against a 25-widget dashboard. Scoring is unchanged -- DashboardSummaryEvaluator grades free text against must_include/must_not_include/rubric and does not care about the transport, so the existing fixtures port over by moving the dashboard id from summary_input to the same field this kind reads. Notes on the shape: - Widgets are executed once and their result ids reused across K runs; they name cached executions, so re-running per K would multiply the item's dominant cost. - A widget that fails to execute is left out of the context (the skill would drop it anyway) but still counted, and detail carries widgets_executed/widgets_total -- a rubric naming a widget that never ran fails for a reason that is not the agent's, and that has to be visible without opening the trace. - max_widgets bounds the executions for wide dashboards. - The dashboard entity is read over plain HTTP: entities_api validates the entity's timestamps with a regex and raises TypeError on a datetime, so the typed accessor cannot read a dashboard at all today. - A ChatError is recorded on the run rather than raised, so it cannot discard the K-runs already completed. Co-Authored-By: Claude Opus 5 --- .../src/gooddata_eval/cli/agentic_runner.py | 19 + .../core/agentic/dashboard_summary.py | 499 ++++++++++++++++++ .../tests/test_agentic_dashboard_summary.py | 272 ++++++++++ .../tests/test_agentic_runner.py | 11 + .../gooddata-eval/tests/test_trace_linker.py | 1 + 5 files changed, 802 insertions(+) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py create mode 100644 packages/gooddata-eval/tests/test_agentic_dashboard_summary.py diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 0bd6f5cf8..e821737d1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -12,6 +12,7 @@ from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation +from gooddata_eval.core.agentic.dashboard_summary import evaluate_agentic_dashboard_summary from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill @@ -44,6 +45,7 @@ class _LfKw(TypedDict, total=False): "agentic_guardrail", "agentic_conversation", "agentic_kda_skill", + "agentic_dashboard_summary", } ) @@ -228,6 +230,23 @@ def _dispatch_agentic( agent_id=agent_id, **lf_kw, ) + elif kind == "agentic_dashboard_summary": + summary_input = item.summary_input + if summary_input is None: + raise ValueError(f"agentic_dashboard_summary item '{item.id}' is missing required 'summary_input'.") + return evaluate_agentic_dashboard_summary( + host=host, + token=token, + workspace_id=workspace_id, + dashboard_id=summary_input.dashboard_id, + expected_output=eo, + # The fixture's own wording is the prompt under test -- a localized fixture is + # only meaningful if its own phrasing is what reaches the agent. + question=item.question, + k=k, + agent_id=agent_id, + **lf_kw, + ) elif kind == "agentic_conversation": fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {} return evaluate_agentic_conversation( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py new file mode 100644 index 000000000..941193866 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py @@ -0,0 +1,499 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Agentic dashboard-summary evaluation runner. + +Covers the path a user actually takes: the dashboard's "Summarize" menu item drops them +into the assistant with "Summarize this dashboard" pre-filled, so the summary is produced +by the conversational ``dashboard_summary`` skill, not by the dedicated +``POST /api/v1/ai/workspaces/{ws}/summary`` endpoint the ``dashboard_summary`` test kind +exercises. That endpoint sits behind its own feature flag (``ENABLE_GEN_AI_HEADLESS_SUMMARY``) +and serves API/embedding consumers; the two share an idea and nothing else. + +The skill reads the dashboard from ``userContext.view.dashboard`` and collects only the +widgets that carry a ``result_id`` -- a widget without one is dropped from the summarize +scope entirely, which is why sending a bare dashboard id gets "no dashboard charts were +provided" and sending descriptors without results gets "these visualizations need to be +reloaded". In the browser the ids exist because the client has already rendered the +widgets. Here we do the same thing deliberately: walk the dashboard's layout, execute each +insight, and hand back the result ids that execution produced. + +Scoring is unchanged from the single-shot kind -- ``DashboardSummaryEvaluator`` grades free +text against the fixture's ``must_include``/``must_not_include``/``rubric``, so it does not +care which transport produced the summary. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + +import httpx +from gooddata_sdk import GoodDataSdk + +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.chat.sse_client import ChatClient, ChatError +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.evaluators.summary import DashboardSummaryEvaluator +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + DatasetItem, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, +) +from gooddata_eval.core.timing import PhaseTimings, log_timer + +_DEFAULT_K = 1 + +# What the "Summarize" menu item pre-fills. Kept verbatim rather than taken from the +# fixture: the fixture's own question is the localized/paraphrased prompt under test, and +# every fixture must still reach the same skill. +_DEFAULT_PROMPT = "Summarize this dashboard" + + +@dataclass +class DashboardWidget: + """One insight widget of a dashboard, as the assistant needs to see it.""" + + widget_id: str + title: str + visualization_id: str + # None when the execution failed. Mirrors the browser, where a widget that did not + # render carries no result and is therefore left out of the summarize scope. + result_id: str | None = None + + +def _insight_widgets(content: dict) -> list[DashboardWidget]: + """Every insight widget in a dashboard's layout, in document order. + + Walks the whole document rather than a fixed path: dashboards nest widgets in + sections, and tab-based ones nest those again under ``tabs``, so the depth is not + known ahead of time. Widgets whose insight carries no identifier are skipped -- a + rich-text or unresolved widget has nothing to execute. + """ + widgets: list[DashboardWidget] = [] + seen: set[str] = set() + + def walk(node: Any) -> None: + if isinstance(node, dict): + insight = node.get("insight") + if isinstance(insight, dict): + identifier = insight.get("identifier") + viz_id = identifier.get("id") if isinstance(identifier, dict) else None + if viz_id: + widget_id = str(node.get("localIdentifier") or viz_id) + if widget_id not in seen: + seen.add(widget_id) + widgets.append( + DashboardWidget( + widget_id=widget_id, + title=str(node.get("title") or viz_id), + visualization_id=str(viz_id), + ) + ) + for value in node.values(): + walk(value) + elif isinstance(node, list): + for value in node: + walk(value) + + walk(content) + return widgets + + +def _execute_widget(sdk: GoodDataSdk, workspace_id: str, visualization_id: str) -> str: + """Execute one saved visualization and return the result id its execution produced. + + Reuses the table service's own pivot/non-pivot split so the execution matches what the + client would run for that visualization; only the result id is wanted, not the data. + """ + import gooddata_sdk.table as table_module # noqa: PLC0415 -- private helpers, imported at use site + + visualization = sdk.visualizations.get_visualization(workspace_id, visualization_id) + is_pivot = table_module._vis_is_table(visualization) or visualization.has_bucket_of_type( + table_module.BucketType.ROWS + ) + exec_def = ( + table_module._get_exec_for_pivot(visualization) + if is_pivot + else table_module.get_exec_for_non_pivot(visualization) + ) + return sdk.compute.for_exec_def(workspace_id, exec_def).result_id + + +def _fetch_dashboard(host: str, token: str, workspace_id: str, dashboard_id: str) -> tuple[str | None, dict]: + """The dashboard's title and layout document. + + Read over plain HTTP rather than through ``entities_api``: the generated client + validates the entity's timestamp fields with a regex and raises + ``TypeError: expected string or bytes-like object, got 'datetime.datetime'`` on the + response, so the typed accessor cannot read a dashboard at all today. + """ + url = f"{host.rstrip('/')}/api/v1/entities/workspaces/{workspace_id}/analyticalDashboards/{dashboard_id}" + resp = httpx.get(url, headers={"Authorization": f"Bearer {token}"}, timeout=120.0) + resp.raise_for_status() + attributes = resp.json()["data"]["attributes"] + return attributes.get("title"), attributes.get("content") or {} + + +def build_dashboard_user_context( + sdk: GoodDataSdk, + host: str, + token: str, + workspace_id: str, + dashboard_id: str, + *, + max_widgets: int | None = None, +) -> tuple[dict, list[DashboardWidget]]: + """Assemble the ``userContext`` for one dashboard, executing its widgets to get result ids. + + Returns the context and the widgets it describes, each carrying the result id its + execution produced or None if that execution failed. A failed widget is still returned + (reporting needs to know the summary was partial) but is left out of the context, since + the skill would drop it anyway. + + ``max_widgets`` caps how many widgets are executed. A dashboard of thirty widgets costs + thirty executions per run, so a fixture that only asserts on headline figures can bound + the cost -- at the price of summarizing less than the user would see. + """ + title, content = _fetch_dashboard(host, token, workspace_id, dashboard_id) + + widgets = _insight_widgets(content) + if max_widgets is not None: + widgets = widgets[:max_widgets] + + for widget in widgets: + try: + widget.result_id = _execute_widget(sdk, workspace_id, widget.visualization_id) + except Exception as exc: # noqa: BLE001, PERF203 -- one dead widget must not lose the whole summary + log_timer(f"[dashboard_summary] widget '{widget.widget_id}' failed to execute: {exc}") + + context = { + "view": { + "dashboard": { + "id": dashboard_id, + "title": title, + "widgets": [ + { + "widgetId": w.widget_id, + "title": w.title, + "widgetType": "insight", + "visualizationId": w.visualization_id, + "resultId": w.result_id, + } + for w in widgets + if w.result_id is not None + ], + } + } + } + return context, widgets + + +@dataclass +class DashboardSummaryRunResult: + """Outcome of one K-run conversation for a dashboard summary.""" + + conversation_id: str + actual_output: str + evaluation: ItemEvaluation + widgets_total: int + widgets_executed: int + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None + timings: PhaseTimings = field(default_factory=PhaseTimings) + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) + # Set when the chat call itself failed. Such a run has no summary to grade, so it is + # recorded rather than raised -- raising would discard the K-runs already completed. + chat_error: str | None = None + + @property + def passed(self) -> bool: + return self.chat_error is None and self.evaluation.passed + + +@dataclass +class AgenticDashboardSummarySummary: + """Aggregated outcome of K runs for one dashboard-summary item.""" + + run_results: list[DashboardSummaryRunResult] + pass_at_k: bool + pass_power_k: bool + best: DashboardSummaryRunResult + + +def _failed_run(conversation_id: str, message: str, widgets_total: int, widgets_executed: int, agent_s: float): + return DashboardSummaryRunResult( + conversation_id=conversation_id, + actual_output="", + evaluation=ItemEvaluation(passed=False, rank_key=(-1, 0.0), detail={}, error=message), + widgets_total=widgets_total, + widgets_executed=widgets_executed, + timings=PhaseTimings(agent_s=agent_s), + chat_error=message, + ) + + +def _run_single_dashboard_summary( + client: ChatClient, + evaluator: DashboardSummaryEvaluator, + conversation_id: str, + item: DatasetItem, + user_context: dict, + widgets: list[DashboardWidget], + prompt: str, +) -> DashboardSummaryRunResult: + widgets_total = len(widgets) + widgets_executed = sum(1 for w in widgets if w.result_id is not None) + + agent_started = time.monotonic() + try: + chat_result = client.send_message(conversation_id, prompt, user_context=user_context) + except ChatError as exc: + return _failed_run( + conversation_id, f"chat failed: {exc}", widgets_total, widgets_executed, time.monotonic() - agent_started + ) + agent_elapsed = time.monotonic() - agent_started + + judge_started = time.monotonic() + evaluation = evaluator.evaluate(item, chat_result) + judge_elapsed = time.monotonic() - judge_started + log_timer( + f"[timer] dashboard_summary {conversation_id} agent {agent_elapsed:.2f}s, judge {judge_elapsed:.2f}s " + f"({widgets_executed}/{widgets_total} widgets executed)" + ) + + return DashboardSummaryRunResult( + conversation_id=conversation_id, + actual_output=str(evaluation.detail.get("actual_output", "")), + evaluation=evaluation, + widgets_total=widgets_total, + widgets_executed=widgets_executed, + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, + timings=PhaseTimings(agent_s=agent_elapsed, judge_s=judge_elapsed), + tool_call_events=list(chat_result.tool_call_events or []), + reasoning_step_events=list(chat_result.reasoning_step_events or []), + ) + + +def run_agentic_dashboard_summary( + host: str, + token: str, + workspace_id: str, + dashboard_id: str, + expected_output: Any, + question: str = _DEFAULT_PROMPT, + k: int = _DEFAULT_K, + max_iterations: int = 1, # noqa: ARG001 -- single-turn kind; accepted so the runner can dispatch uniformly + initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, + max_widgets: int | None = None, + dataset_item_id: str = "", + dataset_name: str = "dashboard_summary", +) -> AgenticDashboardSummarySummary: + """Run the agentic dashboard-summary evaluation K times and return a summary. + + The widgets are executed ONCE and their result ids reused across all K runs: they + identify cached executions, so re-running them per K would multiply the most expensive + part of the item without changing what the assistant sees. + """ + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) + sdk = GoodDataSdk.create(host, token) + evaluator = DashboardSummaryEvaluator() + item = DatasetItem( + id=dataset_item_id or dashboard_id, + dataset_name=dataset_name, + test_kind="agentic_dashboard_summary", + question=question, + expected_output=expected_output, + ) + run_results: list[DashboardSummaryRunResult] = [] + + try: + context_started = time.monotonic() + user_context, widgets = build_dashboard_user_context( + sdk, host, token, workspace_id, dashboard_id, max_widgets=max_widgets + ) + log_timer( + f"[timer] dashboard_summary {dashboard_id} context built in " + f"{time.monotonic() - context_started:.2f}s ({len(widgets)} widgets)" + ) + + conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() + try: + run_results.append( + _run_single_dashboard_summary(client, evaluator, conv_id_0, item, user_context, widgets, question) + ) + finally: + if initial_conversation_id is None: + client.delete_conversation(conv_id_0) + + for _ in range(1, k): + conv_id = client.create_conversation() + try: + run_results.append( + _run_single_dashboard_summary(client, evaluator, conv_id, item, user_context, widgets, question) + ) + finally: + client.delete_conversation(conv_id) + finally: + client.close() + + scored = [r for r in run_results if r.chat_error is None] + pass_at_k = any(r.passed for r in scored) + pass_power_k = len(scored) == len(run_results) and bool(scored) and all(r.passed for r in scored) + best = max(scored or run_results, key=lambda r: r.evaluation.rank_key) + return AgenticDashboardSummarySummary( + run_results=run_results, + pass_at_k=pass_at_k, + pass_power_k=pass_power_k, + best=best, + ) + + +def _detail(summary: AgenticDashboardSummarySummary) -> dict: + best = summary.best + detail = dict(best.evaluation.detail) + # How much of the dashboard the summary actually covered. A summary graded against a + # rubric that mentions a widget which never executed fails for a reason that has + # nothing to do with the agent, so the ratio has to be visible in the report. + detail["widgets_total"] = best.widgets_total + detail["widgets_executed"] = best.widgets_executed + if best.chat_error is not None: + detail["chat_error"] = best.chat_error + detail["latency_breakdown"] = build_latency_breakdown(best.tool_call_events, best.reasoning_step_events) + return detail + + +class DashboardSummaryAssertionError(AgenticAssertionError): + """Raised when an agentic dashboard-summary evaluation fails.""" + + +def evaluate_agentic_dashboard_summary( + host: str, + token: str, + workspace_id: str, + dashboard_id: str, + expected_output: Any, + question: str = _DEFAULT_PROMPT, + k: int = _DEFAULT_K, + initial_conversation_id: str | None = None, + agent_id: str | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "dashboard_summary", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + max_widgets: int | None = None, +) -> AgenticEvalOutcome: + """Run the evaluation, log to Langfuse, and raise DashboardSummaryAssertionError on failure.""" + langfuse, window_start = open_trace_window(langfuse) + summary = run_agentic_dashboard_summary( + host=host, + token=token, + workspace_id=workspace_id, + dashboard_id=dashboard_id, + expected_output=expected_output, + question=question, + k=k, + initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + max_widgets=max_widgets, + dataset_item_id=dataset_item_id, + dataset_name=dataset_name, + ) + + if langfuse is not None and dataset_item_id: + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + for run_idx, run in enumerate(summary.run_results): + if run.chat_error is not None: + # The chat never produced a summary, so there is no verdict: writing + # 0.0 would publish a content failure the judge never assessed. + continue + pt = ctx.trace(run.conversation_id) + with ctx.observe(pt, run_idx) as tid: + ctx.score( + tid, name="dashboard_summary_pass", value=float(run.evaluation.passed), data_type="BOOLEAN" + ) + ctx.quality( + tid, + strict_checks={"dashboard_summary_pass": run.evaluation.passed}, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + # A run whose chat failed is skipped above, so polling for its trace would only + # spend the item's shared retry budget on scores that never get written. + conversation_ids=[r.conversation_id for r in summary.run_results if r.chat_error is None], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, + ) + + best = summary.best + detail = _detail(summary) + timings = PhaseTimings() + for run in summary.run_results: + timings = timings + run.timings + runs_passed = sum(1 for r in summary.run_results if r.passed) + + if not summary.pass_at_k: + error = DashboardSummaryAssertionError( + f"Dashboard summary failed for '{dashboard_id}' " + f"({best.widgets_executed}/{best.widgets_total} widgets executed): " + f"{best.evaluation.error or 'criteria not satisfied'}" + ) + error.reasoning_steps = best.reasoning_steps + error.conversation_id = best.conversation_id + error.response_id = best.response_id + error.detail = detail + error.timings = timings + error.runs_passed = runs_passed + error.runs_effective = len(summary.run_results) + raise error + + return AgenticEvalOutcome( + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + timings=timings, + runs_passed=runs_passed, + runs_effective=len(summary.run_results), + ) diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py b/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py new file mode 100644 index 000000000..5656a8482 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py @@ -0,0 +1,272 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic.dashboard_summary import ( + DashboardSummaryAssertionError, + DashboardWidget, + _insight_widgets, + build_dashboard_user_context, + evaluate_agentic_dashboard_summary, + run_agentic_dashboard_summary, +) +from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.models import ChatResult + +_MODULE = "gooddata_eval.core.agentic.dashboard_summary" + + +def _widget(local_id: str, viz_id: str, title: str = "A chart") -> dict: + return { + "localIdentifier": local_id, + "title": title, + "insight": {"identifier": {"id": viz_id, "type": "visualizationObject"}}, + } + + +def _summary_result(text: str = "## Executive Summary\n- Approvals trended up.") -> ChatResult: + return ChatResult.model_validate({"textResponse": text, "toolCallEvents": [], "reasoningSteps": []}) + + +# ── layout walking ────────────────────────────────────────────────────────── + + +def test_insight_widgets_walks_nested_sections_and_tabs(): + """Widget depth is not fixed: sections nest widgets, and a tab-based dashboard nests + those again, so the walk cannot assume a path.""" + content = { + "version": "3", + "tabs": [ + {"localIdentifier": "tab1", "sections": [{"items": [_widget("w1", "viz-1")]}]}, + {"localIdentifier": "tab2", "sections": [{"items": [_widget("w2", "viz-2"), _widget("w3", "viz-3")]}]}, + ], + } + assert [w.widget_id for w in _insight_widgets(content)] == ["w1", "w2", "w3"] + assert [w.visualization_id for w in _insight_widgets(content)] == ["viz-1", "viz-2", "viz-3"] + + +def test_insight_widgets_skips_widgets_with_nothing_to_execute(): + """Rich text and unresolved widgets carry no insight identifier, so there is nothing to + execute and nothing the summarize scope could use.""" + content = { + "sections": [ + {"items": [_widget("w1", "viz-1"), {"localIdentifier": "rt", "richText": {"content": "hello"}}]}, + {"items": [{"localIdentifier": "broken", "insight": {}}]}, + ] + } + assert [w.widget_id for w in _insight_widgets(content)] == ["w1"] + + +def test_insight_widgets_does_not_repeat_a_widget_reached_twice(): + """The walk descends into every value, so a layout that references one widget from two + places must still yield it once -- a duplicate would be executed twice.""" + shared = _widget("w1", "viz-1") + content = {"sections": [{"items": [shared]}], "alsoHere": {"items": [shared]}} + assert [w.widget_id for w in _insight_widgets(content)] == ["w1"] + + +def test_insight_widget_falls_back_to_the_visualization_id_when_unnamed(): + content = {"items": [{"insight": {"identifier": {"id": "viz-1"}}}]} + (widget,) = _insight_widgets(content) + assert widget.widget_id == "viz-1" + assert widget.title == "viz-1" + + +# ── context building ──────────────────────────────────────────────────────── + + +def _patched_dashboard(widgets: list[dict], title: str = "Sales"): + return patch(f"{_MODULE}._fetch_dashboard", return_value=(title, {"sections": [{"items": widgets}]})) + + +def test_build_user_context_carries_a_result_id_per_widget(): + sdk = MagicMock() + with ( + _patched_dashboard([_widget("w1", "viz-1"), _widget("w2", "viz-2")]), + patch(f"{_MODULE}._execute_widget", side_effect=["res-1", "res-2"]), + ): + context, widgets = build_dashboard_user_context(sdk, "http://h", "tok", "ws1", "dash-1") + + dashboard = context["view"]["dashboard"] + assert dashboard["id"] == "dash-1" + assert dashboard["title"] == "Sales" + assert [w["resultId"] for w in dashboard["widgets"]] == ["res-1", "res-2"] + assert [w["widgetType"] for w in dashboard["widgets"]] == ["insight", "insight"] + assert [w.result_id for w in widgets] == ["res-1", "res-2"] + + +def test_a_widget_that_fails_to_execute_is_reported_but_left_out_of_the_context(): + """The skill drops any widget without a result_id, so sending one would be a silent + no-op -- but the report still has to show the summary covered less than the dashboard.""" + sdk = MagicMock() + with ( + _patched_dashboard([_widget("w1", "viz-1"), _widget("w2", "viz-2")]), + patch(f"{_MODULE}._execute_widget", side_effect=["res-1", RuntimeError("execution blew up")]), + ): + context, widgets = build_dashboard_user_context(sdk, "http://h", "tok", "ws1", "dash-1") + + assert [w["widgetId"] for w in context["view"]["dashboard"]["widgets"]] == ["w1"] + assert len(widgets) == 2 + assert widgets[1].result_id is None + + +def test_max_widgets_caps_the_executions(): + """A thirty-widget dashboard costs thirty executions per item; the cap bounds that.""" + sdk = MagicMock() + layout = [_widget(f"w{i}", f"viz-{i}") for i in range(5)] + with ( + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res") as execute, + ): + _, widgets = build_dashboard_user_context(sdk, "http://h", "tok", "ws1", "dash-1", max_widgets=2) + + assert len(widgets) == 2 + assert execute.call_count == 2 + + +# ── runs ──────────────────────────────────────────────────────────────────── + + +def _run(chat_side_effect, *, k=1, passed=True, widgets=2): + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, k + 2)] + client.send_message.side_effect = chat_side_effect + evaluation = ItemEvaluation(passed=passed, rank_key=(int(passed), 1.0), detail={"actual_output": "text"}) + evaluator = MagicMock() + evaluator.evaluate.return_value = evaluation + layout = [_widget(f"w{i}", f"viz-{i}") for i in range(widgets)] + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res"), + ): + return run_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + k=k, + ) + + +def test_a_passing_run_records_how_much_of_the_dashboard_it_covered(): + summary = _run([_summary_result()]) + assert summary.pass_at_k is True + assert summary.best.widgets_total == 2 + assert summary.best.widgets_executed == 2 + + +def test_widgets_are_executed_once_and_reused_across_k_runs(): + """Result ids identify cached executions, so re-running them per K would multiply the + most expensive part of the item without changing what the assistant sees.""" + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2", "conv-3"] + client.send_message.return_value = _summary_result() + evaluator = MagicMock() + evaluator.evaluate.return_value = ItemEvaluation(passed=True, rank_key=(1, 1.0), detail={"actual_output": "t"}) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard([_widget("w1", "viz-1")]), + patch(f"{_MODULE}._execute_widget", return_value="res") as execute, + ): + summary = run_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + k=3, + ) + + assert len(summary.run_results) == 3 + assert execute.call_count == 1 + + +def test_a_chat_error_is_recorded_on_the_run_rather_than_raised(): + """Raising would discard the K-runs already completed and leave the item with no + verdict at all -- the same failure mode fixed for the other agentic kinds.""" + summary = _run([ChatError("gen-ai fell over")]) + + assert summary.pass_at_k is False + run = summary.run_results[0] + assert run.chat_error is not None + assert "gen-ai fell over" in run.chat_error + assert run.passed is False + assert run.evaluation.error is not None + + +def test_a_chat_error_on_a_later_run_does_not_discard_the_earlier_one(): + summary = _run([_summary_result(), ChatError("boom")], k=2) + + assert len(summary.run_results) == 2 + assert summary.run_results[0].chat_error is None + assert summary.run_results[1].chat_error is not None + assert summary.pass_at_k is True # run 0 still counts + # The chat-error run is unscored, so it cannot certify that every run passed. + assert summary.pass_power_k is False + + +def test_the_best_run_is_never_a_chat_error_when_a_graded_run_exists(): + summary = _run([ChatError("boom"), _summary_result()], k=2) + assert summary.best.chat_error is None + + +# ── evaluate_* wrapper ────────────────────────────────────────────────────── + + +def _evaluate(passed: bool): + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = _summary_result() + evaluator = MagicMock() + evaluator.evaluate.return_value = ItemEvaluation( + passed=passed, rank_key=(int(passed), 1.0), detail={"actual_output": "text", "include_0": passed} + ) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard([_widget("w1", "viz-1"), _widget("w2", "viz-2")]), + patch(f"{_MODULE}._execute_widget", side_effect=["res-1", RuntimeError("dead")]), + ): + return evaluate_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + ) + + +def test_detail_reports_coverage_and_the_latency_breakdown(): + outcome = _evaluate(passed=True) + # A rubric naming a widget that never executed fails for a reason that is not the + # agent's, so the ratio has to be visible without opening the trace. + assert outcome.detail["widgets_total"] == 2 + assert outcome.detail["widgets_executed"] == 1 + assert "latency_breakdown" in outcome.detail + assert outcome.detail["include_0"] is True + assert outcome.conversation_id == "conv-1" + + +def test_a_failing_item_raises_with_the_same_detail_attached(): + with pytest.raises(DashboardSummaryAssertionError) as exc_info: + _evaluate(passed=False) + + error = exc_info.value + assert "1/2 widgets executed" in str(error) + assert error.detail["widgets_executed"] == 1 + assert error.runs_passed == 0 + assert error.runs_effective == 1 + assert error.conversation_id == "conv-1" + + +def test_dashboard_widget_defaults_to_no_result(): + assert DashboardWidget(widget_id="w", title="t", visualization_id="v").result_id is None diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 77086d101..cf2eecdcb 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -85,8 +85,17 @@ def test_dispatch_agentic_omits_agent_id_by_default(): ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), ("agentic_kda_skill", {"Measure": {"type": "metric", "id": "revenue"}}, "evaluate_agentic_kda_skill"), ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), + ( + "agentic_dashboard_summary", + {"must_include": ["States the overall trend."]}, + "evaluate_agentic_dashboard_summary", + ), ] +# The one kind whose dispatch needs more than question/expected_output: the dashboard to +# summarize is named in summary_input, exactly as for the single-shot dashboard_summary. +_SUMMARY_INPUT_KINDS = {"agentic_dashboard_summary"} + def test_all_agentic_kind_cases_covers_every_registered_kind(): """Guards the two parametrized tests below against silently going stale: a kind added @@ -104,6 +113,7 @@ def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_ test_kind=kind, question="q", expected_output=expected_output, + summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, ) with patch(f"gooddata_eval.cli.agentic_runner.{target}") as mock_eval: _dispatch_agentic( @@ -209,6 +219,7 @@ def test_dispatch_agentic_returns_a_real_outcome_for_every_kind(kind, expected_o test_kind=kind, question="q", expected_output=expected_output, + summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, ) canned = AgenticEvalOutcome(reasoning_steps=["x"], conversation_id="c1", response_id="r1", detail={"k": "v"}) with patch(f"gooddata_eval.cli.agentic_runner.{target}", return_value=canned) as mock_eval: diff --git a/packages/gooddata-eval/tests/test_trace_linker.py b/packages/gooddata-eval/tests/test_trace_linker.py index 4737ee34c..5d6279f29 100644 --- a/packages/gooddata-eval/tests/test_trace_linker.py +++ b/packages/gooddata-eval/tests/test_trace_linker.py @@ -118,6 +118,7 @@ def test_run_trace_link_inline_runs_the_task_on_the_calling_thread(): ("visualization", "evaluate_agentic_visualization"), ("kda_skill", "evaluate_agentic_kda_skill"), ("conversation", "evaluate_agentic_conversation"), + ("dashboard_summary", "evaluate_agentic_dashboard_summary"), ] From a86cd4275542ba1a4d326bfdcf0a3061f943acde Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 10 Sep 2026 00:08:26 +0200 Subject: [PATCH 2/3] fix(gooddata-eval): address review on the agentic dashboard-summary kind Four points from review, three of them behavioural. create_conversation() in the K-run loop sat outside any handler, so a transient failure on run 2 of 3 discarded run 1 -- the exact contract the ChatError path next to it exists to keep. Recorded as a failed run instead. max_widgets was documented as the way to bound execution cost but the CLI dispatch never passed it, so every fixture executed the whole dashboard: thirty widgets is thirty executions per item. Fixtures now bound it by naming the charts they assert on, through summary_input.visualizations -- the same field and meaning the headless /summary endpoint gives it, and stable as a dashboard grows in a way a positional cap is not. max_widgets stays as a blunt cap for direct callers. _execute_widget borrows two private helpers from gooddata_sdk.table under a ~=1.74.0 dependency, so a patch release may rename them. It now checks for all three up front and raises naming the dependency, rather than surfacing an AttributeError on the first widget of a run. The docstring also records why the public sdk.tables.for_visualization is not used: it reads the whole result into an ExecutionTable and returns that, discarding the result id this needs and paying for every row to do it. _DEFAULT_PROMPT's comment claimed the prompt is kept verbatim rather than taken from the fixture, while the dispatch passes item.question with the opposite rationale written beside it. The dispatch is right -- a localized fixture is only meaningful if its own wording reaches the agent -- so the comment now describes the constant as the default for direct callers. Five tests added; the three behavioural ones verified to fail against the previous version. 800 passed, lint and format clean. Co-Authored-By: Claude Opus 5 --- .../src/gooddata_eval/cli/agentic_runner.py | 4 + .../core/agentic/dashboard_summary.py | 70 +++++++++++--- .../tests/test_agentic_dashboard_summary.py | 95 +++++++++++++++++++ 3 files changed, 155 insertions(+), 14 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index e821737d1..fdacad929 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -243,6 +243,10 @@ def _dispatch_agentic( # The fixture's own wording is the prompt under test -- a localized fixture is # only meaningful if its own phrasing is what reaches the agent. question=item.question, + # Each widget costs an execution, so a fixture asserting on a handful of charts + # can name them instead of paying for the whole dashboard. Same field and meaning + # the headless /summary endpoint gives it; None summarizes everything. + only_visualizations=summary_input.visualizations, k=k, agent_id=agent_id, **lf_kw, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py index 941193866..9edd72707 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py @@ -55,9 +55,9 @@ _DEFAULT_K = 1 -# What the "Summarize" menu item pre-fills. Kept verbatim rather than taken from the -# fixture: the fixture's own question is the localized/paraphrased prompt under test, and -# every fixture must still reach the same skill. +# What the "Summarize" menu item pre-fills, used when no question is supplied. A fixture's +# own question wins: it is the localized or paraphrased wording under test, and the dispatch +# passes it through for exactly that reason. This is the default for direct callers only. _DEFAULT_PROMPT = "Summarize this dashboard" @@ -116,9 +116,28 @@ def _execute_widget(sdk: GoodDataSdk, workspace_id: str, visualization_id: str) Reuses the table service's own pivot/non-pivot split so the execution matches what the client would run for that visualization; only the result id is wanted, not the data. + + ``sdk.tables.for_visualization`` is the public equivalent, but it reads the whole result + into an ExecutionTable and returns that instead of the response -- which discards the one + field wanted here and pays for every row to do it. Two of the three helpers it delegates + to are private, so they are checked up front: ``gooddata-sdk`` is depended on as + ``~=1.74.0`` and a patch release may rename them, which would otherwise surface as an + AttributeError on the first widget of a run rather than as a dependency problem. """ import gooddata_sdk.table as table_module # noqa: PLC0415 -- private helpers, imported at use site + missing = [ + name + for name in ("_vis_is_table", "_get_exec_for_pivot", "get_exec_for_non_pivot") + if not hasattr(table_module, name) + ] + if missing: + raise RuntimeError( + f"gooddata_sdk.table no longer provides {', '.join(missing)}, so a dashboard widget cannot be " + "executed for its result id. The installed gooddata-sdk has moved these helpers -- either pin it " + "back or port _execute_widget onto whatever replaced them." + ) + visualization = sdk.visualizations.get_visualization(workspace_id, visualization_id) is_pivot = table_module._vis_is_table(visualization) or visualization.has_bucket_of_type( table_module.BucketType.ROWS @@ -153,6 +172,7 @@ def build_dashboard_user_context( workspace_id: str, dashboard_id: str, *, + only_visualizations: list[str] | None = None, max_widgets: int | None = None, ) -> tuple[dict, list[DashboardWidget]]: """Assemble the ``userContext`` for one dashboard, executing its widgets to get result ids. @@ -162,13 +182,19 @@ def build_dashboard_user_context( (reporting needs to know the summary was partial) but is left out of the context, since the skill would drop it anyway. - ``max_widgets`` caps how many widgets are executed. A dashboard of thirty widgets costs - thirty executions per run, so a fixture that only asserts on headline figures can bound - the cost -- at the price of summarizing less than the user would see. + A dashboard of thirty widgets costs thirty executions per item, so both arguments exist + to bound that -- at the price of summarizing less than the user would see. + ``only_visualizations`` keeps just the named visualizations, which is what a fixture sets + (via ``summary_input.visualizations``, the same field and meaning the headless endpoint + gives it) because naming the widgets a rubric asserts on stays stable as the dashboard + grows. ``max_widgets`` truncates in layout order and is a blunt cap for direct callers. """ title, content = _fetch_dashboard(host, token, workspace_id, dashboard_id) widgets = _insight_widgets(content) + if only_visualizations is not None: + wanted = set(only_visualizations) + widgets = [w for w in widgets if w.visualization_id in wanted or w.widget_id in wanted] if max_widgets is not None: widgets = widgets[:max_widgets] @@ -233,13 +259,16 @@ class AgenticDashboardSummarySummary: best: DashboardSummaryRunResult -def _failed_run(conversation_id: str, message: str, widgets_total: int, widgets_executed: int, agent_s: float): +def _failed_run( + conversation_id: str, message: str, widgets: list[DashboardWidget], agent_s: float +) -> DashboardSummaryRunResult: + """A run that never produced a summary, recorded so the completed runs survive.""" return DashboardSummaryRunResult( conversation_id=conversation_id, actual_output="", evaluation=ItemEvaluation(passed=False, rank_key=(-1, 0.0), detail={}, error=message), - widgets_total=widgets_total, - widgets_executed=widgets_executed, + widgets_total=len(widgets), + widgets_executed=sum(1 for w in widgets if w.result_id is not None), timings=PhaseTimings(agent_s=agent_s), chat_error=message, ) @@ -261,9 +290,7 @@ def _run_single_dashboard_summary( try: chat_result = client.send_message(conversation_id, prompt, user_context=user_context) except ChatError as exc: - return _failed_run( - conversation_id, f"chat failed: {exc}", widgets_total, widgets_executed, time.monotonic() - agent_started - ) + return _failed_run(conversation_id, f"chat failed: {exc}", widgets, time.monotonic() - agent_started) agent_elapsed = time.monotonic() - agent_started judge_started = time.monotonic() @@ -300,6 +327,7 @@ def run_agentic_dashboard_summary( initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, + only_visualizations: list[str] | None = None, max_widgets: int | None = None, dataset_item_id: str = "", dataset_name: str = "dashboard_summary", @@ -327,7 +355,13 @@ def run_agentic_dashboard_summary( try: context_started = time.monotonic() user_context, widgets = build_dashboard_user_context( - sdk, host, token, workspace_id, dashboard_id, max_widgets=max_widgets + sdk, + host, + token, + workspace_id, + dashboard_id, + only_visualizations=only_visualizations, + max_widgets=max_widgets, ) log_timer( f"[timer] dashboard_summary {dashboard_id} context built in " @@ -344,7 +378,13 @@ def run_agentic_dashboard_summary( client.delete_conversation(conv_id_0) for _ in range(1, k): - conv_id = client.create_conversation() + try: + conv_id = client.create_conversation() + except Exception as exc: # noqa: BLE001 -- a lost conversation must not discard completed runs + # Same contract as the ChatError path below: this line used to sit outside any + # handler, so a transient failure here on run 2 of 3 threw away run 1. + run_results.append(_failed_run("", f"conversation creation failed: {exc}", widgets, 0.0)) + continue try: run_results.append( _run_single_dashboard_summary(client, evaluator, conv_id, item, user_context, widgets, question) @@ -402,6 +442,7 @@ def evaluate_agentic_dashboard_summary( run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, + only_visualizations: list[str] | None = None, max_widgets: int | None = None, ) -> AgenticEvalOutcome: """Run the evaluation, log to Langfuse, and raise DashboardSummaryAssertionError on failure.""" @@ -417,6 +458,7 @@ def evaluate_agentic_dashboard_summary( initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, agent_id=agent_id, + only_visualizations=only_visualizations, max_widgets=max_widgets, dataset_item_id=dataset_item_id, dataset_name=dataset_name, diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py b/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py index 5656a8482..431ac50bd 100644 --- a/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py @@ -2,10 +2,13 @@ # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise from unittest.mock import MagicMock, patch +import gooddata_sdk +import gooddata_sdk.table as table_module import pytest from gooddata_eval.core.agentic.dashboard_summary import ( DashboardSummaryAssertionError, DashboardWidget, + _execute_widget, _insight_widgets, build_dashboard_user_context, evaluate_agentic_dashboard_summary, @@ -270,3 +273,95 @@ def test_a_failing_item_raises_with_the_same_detail_attached(): def test_dashboard_widget_defaults_to_no_result(): assert DashboardWidget(widget_id="w", title="t", visualization_id="v").result_id is None + + +def test_only_visualizations_keeps_just_the_named_charts(): + """A fixture asserting on a handful of charts should not pay to execute thirty. Names + are matched against the visualization id and the widget id, since a fixture author + reading a dashboard's AAC sees both.""" + sdk = MagicMock() + layout = [_widget("w0", "viz-0"), _widget("w1", "viz-1"), _widget("w2", "viz-2")] + with ( + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res") as execute, + ): + _, widgets = build_dashboard_user_context( + sdk, "http://h", "tok", "ws1", "dash-1", only_visualizations=["viz-1", "w2"] + ) + + assert [w.widget_id for w in widgets] == ["w1", "w2"] + assert execute.call_count == 2 + + +def test_only_visualizations_and_max_widgets_compose(): + sdk = MagicMock() + layout = [_widget(f"w{i}", f"viz-{i}") for i in range(4)] + with ( + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res"), + ): + _, widgets = build_dashboard_user_context( + sdk, + "http://h", + "tok", + "ws1", + "dash-1", + only_visualizations=["viz-1", "viz-2", "viz-3"], + max_widgets=2, + ) + + assert [w.widget_id for w in widgets] == ["w1", "w2"] + + +def test_a_failed_conversation_creation_does_not_discard_completed_runs(): + """create_conversation sat outside any handler, so a transient failure on run 2 of 3 + threw away run 1 -- the same failure mode the ChatError path already guards.""" + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", RuntimeError("no conversation for you"), "conv-3"] + client.send_message.return_value = _summary_result() + evaluator = MagicMock() + evaluator.evaluate.return_value = ItemEvaluation(passed=True, rank_key=(1, 1.0), detail={"actual_output": "t"}) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard([_widget("w1", "viz-1")]), + patch(f"{_MODULE}._execute_widget", return_value="res"), + ): + summary = run_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + k=3, + ) + + assert len(summary.run_results) == 3 + assert [r.chat_error is None for r in summary.run_results] == [True, False, True] + assert "conversation creation failed" in summary.run_results[1].chat_error + assert summary.pass_at_k is True + # The lost run has no conversation to score, so it must not certify an all-passed item. + assert summary.pass_power_k is False + + +def test_execute_widget_fails_loudly_when_the_sdk_moves_its_private_helpers(): + """gooddata-sdk is depended on as ~=1.74.0, so a patch release may rename the private + helpers this borrows. Without the check that surfaces as an AttributeError on the first + widget of a run rather than as the dependency problem it is.""" + stripped = MagicMock(spec=[]) # a gooddata_sdk.table exposing none of the three helpers + # Both bindings: `import a.b as c` reads the parent package's attribute, while + # sys.modules is what keeps a re-import from restoring the real one. + with ( + patch.dict("sys.modules", {"gooddata_sdk.table": stripped}), + patch.object(gooddata_sdk, "table", stripped), + pytest.raises(RuntimeError, match="no longer provides"), + ): + _execute_widget(MagicMock(), "ws1", "viz-1") + + +def test_execute_widget_guard_passes_against_the_installed_sdk(): + """The guard must not be a permanent tripwire: the helpers exist in the pinned version, + so a real call gets past it and fails (if at all) on the network, not on the check.""" + for name in ("_vis_is_table", "_get_exec_for_pivot", "get_exec_for_non_pivot"): + assert hasattr(table_module, name), f"gooddata_sdk.table.{name} is gone -- update _execute_widget" From 975c392af6c50a02e1262e1f7005751a5418e2fe Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 10 Sep 2026 13:20:41 +0200 Subject: [PATCH 3/3] fix(gooddata-eval): pass item_input when deferring dashboard-summary scores Merges master, which added a guard requiring every kind to hand the scored item's question to the linker -- a score is otherwise readable only by resolving its conversation back to the item. This kind predates the guard and did not. 937 passed, lint and format clean. Co-Authored-By: Claude Opus 5 --- .../src/gooddata_eval/core/agentic/dashboard_summary.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py index 9edd72707..6b5d7b05d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py @@ -506,6 +506,9 @@ def _write_scores(ctx: RunTraceContext) -> None: window_end=window_end, suffix_runs=len(summary.run_results) > 1, write_scores=_write_scores, + # The question this run answered, so a score is readable without resolving the + # conversation back to its item. + item_input=question, ) best = summary.best