fix(gooddata-eval): stop transient gen-ai faults from being scored as content failures

packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py

@@ -32,8 +32,15 @@
 # gen-ai's last event, only if at least one item was already emitted (conversations_controller.py).
 _RESPONSE_ENDED_EVENT = "response_ended"
 
-_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 502, 503, 504})
+# 500 is here on evidence, not on principle: in one visualization eval batch it hard-failed
+# 10 of 56 runs with zero retry attempts, and every affected question scored normally when the
+# same question/model ran again the next day -- i.e. transient gen-ai faults, not deterministic
+# server bugs. A genuinely deterministic 500 still terminates, just after the bounded backoff.
+_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 500, 502, 503, 504})
 _METADATA_SYNC_MARKER = "METADATA_SYNC_IN_PROGRESS"
+# Stands in for the `id` a persisted visualization would carry, on the fallback path
+# where the agent's create_adhoc_visualization call failed and only its arguments survive.
+_ADHOC_VIZ_ID = "adhoc-visualization-not-persisted"
 
 _KNOWN_PART_TYPES: frozenset[str] = frozenset(
     {
@@ -247,8 +254,14 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
         # Fallback: the agent produced a correct visualization definition via
         # create_adhoc_visualization but the call failed (e.g. data source not
         # accessible). The last attempt is the agent's best answer.
+        #
+        # These are raw tool-call arguments, so they carry no `id` -- nothing was
+        # ever persisted. CreatedVisualization requires one, so synthesize a
+        # sentinel rather than letting the whole ChatResult fail to validate:
+        # dropping the turn entirely would score a stalled data source as a
+        # content failure, which is exactly what this fallback exists to prevent.
         payload["createdVisualizations"] = {
-            "objects": [acc.adhoc_viz_args[-1]],
+            "objects": [{"id": _ADHOC_VIZ_ID, **acc.adhoc_viz_args[-1]}],
             "reasoning": "\n".join(acc.viz_reasoning_parts),
         }
     result = ChatResult.model_validate(payload)

packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py

@@ -2,6 +2,7 @@
 """Agentic visualization evaluator — ported from gdc-nas tavern-e2e app/vis_agentic.py."""
 
 from dataclasses import dataclass
+from datetime import date
 
 from gooddata_eval.core.evaluators.base import ItemEvaluation
 from gooddata_eval.core.models import (
@@ -85,7 +86,13 @@ def _evaluate_visualization(
     expected: CreatedVisualization,
     actual: CreatedVisualization | None,
     skill_activated: bool = False,
+    today: date | None = None,
 ) -> EvaluationResult:
+    # One anchor for the scoring and for the filters reported beside it. check_filters
+    # already shares an anchor across its two sides, but resolving the reported filters
+    # separately would let a run that straddles midnight report periods the score was
+    # never computed from -- a detail that contradicts its own verdict.
+    today = today or date.today()
     exp_metric_uris = get_metric_uri_set(expected)
     exp_dim_uris = get_dimension_uri_set(expected)
     if actual is None:
@@ -105,13 +112,13 @@ def _evaluate_visualization(
             actual_metric_uris=set(),
             expected_dim_uris=exp_dim_uris,
             actual_dim_uris=set(),
-            expected_filters=normalized_filters(expected),
+            expected_filters=normalized_filters(expected, today),
             actual_filters={category: values.copy() for category, values in _NO_FILTERS.items()},
         )
     cross_ref_valid, cross_ref_errors = validate_cross_references(actual)
     act_metric_uris = get_metric_uri_set(actual)
     act_dim_uris = get_dimension_uri_set(actual)
-    filter_scores = check_filters(expected, actual)
+    filter_scores = check_filters(expected, actual, today)
     return EvaluationResult(
         visualization_created=True,
         cross_ref_valid=cross_ref_valid,
@@ -128,17 +135,21 @@ def _evaluate_visualization(
         actual_metric_uris=act_metric_uris,
         expected_dim_uris=exp_dim_uris,
         actual_dim_uris=act_dim_uris,
-        expected_filters=normalized_filters(expected),
-        actual_filters=normalized_filters(actual),
+        expected_filters=normalized_filters(expected, today),
+        actual_filters=normalized_filters(actual, today),
     )
 
 
 def _evaluate_against_candidates(
     expected_outputs: list[CreatedVisualization],
     actual: CreatedVisualization | None,
     skill_activated: bool = False,
+    today: date | None = None,
 ) -> tuple[EvaluationResult, CreatedVisualization]:
-    pairs = [(_evaluate_visualization(exp, actual, skill_activated), exp) for exp in expected_outputs]
+    # Anchored once here too: the candidates are ranked against each other, so letting
+    # each resolve its own today could rank them on spans from different days.
+    today = today or date.today()
+    pairs = [(_evaluate_visualization(exp, actual, skill_activated, today), exp) for exp in expected_outputs]
     best_result, best_expected = max(pairs, key=lambda p: (p[0].strict_pass, p[0].strict_checks_passed_count))
     return best_result, best_expected
 

packages/gooddata-eval/src/gooddata_eval/core/scoring.py

@@ -1,8 +1,10 @@
 # (C) 2026 GoodData Corporation
 """Visualization scoring — ported from gdc-nas tavern-e2e app/vis_assertions/metrics.py."""
 
+import calendar
 import json
 from dataclasses import dataclass
+from datetime import date, timedelta
 
 from gooddata_eval.core.models import AacBucketRef, AacQueryField, CreatedVisualization
 
@@ -106,13 +108,94 @@ def validate_cross_references(viz: CreatedVisualization) -> tuple[bool, list[str
     return len(errors) == 0, errors
 
 
-def _normalize_date_filter(filter_dict: dict, _fields: dict) -> dict:
+def _shift_month(anchor: date, offset: int) -> tuple[date, date]:
+    """First and last day of the calendar month ``offset`` months from ``anchor``."""
+    total = anchor.year * 12 + (anchor.month - 1) + offset
+    year, month = divmod(total, 12)
+    return date(year, month + 1, 1), date(year, month + 1, calendar.monthrange(year, month + 1)[1])
+
+
+def _absolute_span(granularity: str, start_offset: int, end_offset: int, today: date) -> tuple[date, date] | None:
+    """Resolve a relative date filter to the inclusive absolute span it denotes.
+
+    Returns None for granularities this cannot resolve unambiguously -- notably a bare
+    ``WEEK``, which is not in the AAC granularity enum and states no start-of-week
+    convention. ``WEEK_US`` *is* well defined (Sunday-start), so it resolves.
+    Guessing the bare case would trade a false negative for a false positive.
+
+    An offset far enough out to leave the representable date range resolves to None
+    rather than raising: every granularity here can be pushed past it (a YEAR offset of
+    ``-today.year`` alone lands on year 0), and letting that escape would abort scoring
+    for the whole item over one malformed filter. None puts the filter back on the
+    literal-comparison path, which is what an unresolvable span already does.
+    """
+    gran = granularity.upper()
+    try:
+        if gran == "DAY":
+            return today + timedelta(days=start_offset), today + timedelta(days=end_offset)
+        if gran == "WEEK_US":
+            sunday = today - timedelta(days=(today.weekday() + 1) % 7)
+            return sunday + timedelta(weeks=start_offset), sunday + timedelta(weeks=end_offset, days=6)
+        if gran == "MONTH":
+            return _shift_month(today, start_offset)[0], _shift_month(today, end_offset)[1]
+        if gran == "QUARTER":
+            q_start_month = (today.month - 1) // 3 * 3 + 1
+            anchor = date(today.year, q_start_month, 1)
+            return _shift_month(anchor, start_offset * 3)[0], _shift_month(anchor, end_offset * 3 + 2)[1]
+        if gran == "YEAR":
+            return date(today.year + start_offset, 1, 1), date(today.year + end_offset, 12, 31)
+    except (ValueError, OverflowError):
+        return None
+    return None
+
+
+def _as_date(value: object) -> date | None:
+    if isinstance(value, str):
+        try:
+            return date.fromisoformat(value[:10])
+        except ValueError:
+            return None
+    return None
+
+
+def _normalize_date_filter(filter_dict: dict, _fields: dict, today: date | None = None) -> dict:
+    """Canonicalize a date filter, resolving relative offsets to an absolute span.
+
+    The agent may answer "last month" either relatively (``granularity: MONTH,
+    from: -1, to: -1``) or absolutely (``from: 2026-08-01, to: 2026-08-31``).
+    Compared literally these never match, so a correct answer in the encoding the
+    fixture did not happen to use was scored as a wrong date period. Both forms
+    collapse to the same absolute span here.
+
+    Resolution is relative to today, which is the same "today" the agent resolved
+    against -- scoring runs in the same process as the turn. Re-scoring an archived
+    result at a later date would therefore drift; nothing currently does that.
+    """
+    raw_from, raw_to = filter_dict.get("from"), filter_dict.get("to")
+    granularity = filter_dict.get("granularity")
+    span: tuple[date, date] | None = None
+
+    if isinstance(raw_from, int) and isinstance(raw_to, int) and isinstance(granularity, str):
+        span = _absolute_span(granularity, raw_from, raw_to, today or date.today())
+    else:
+        start, end = _as_date(raw_from), _as_date(raw_to)
+        if start and end:
+            span = (start, end)
+
+    if span is not None:
+        return {
+            "type": "date_filter",
+            "dataset_uri": filter_dict.get("using", ""),
+            "from": span[0].isoformat(),
+            "to": span[1].isoformat(),
+        }
+    # Unresolvable (e.g. the WEEK family): fall back to literal comparison.
     return {
         "type": "date_filter",
         "dataset_uri": filter_dict.get("using", ""),
-        "from": filter_dict.get("from"),
-        "to": filter_dict.get("to"),
-        "granularity": filter_dict.get("granularity"),
+        "from": raw_from,
+        "to": raw_to,
+        "granularity": granularity,
     }
 
 
@@ -171,7 +254,9 @@ def _normalize_attribute_filter(filter_dict: dict, _fields: dict) -> dict:
     }
 
 
-def _split_and_normalize_filters(viz: CreatedVisualization) -> tuple[set[str], set[str], set[str]]:
+def _split_and_normalize_filters(
+    viz: CreatedVisualization, today: date | None = None
+) -> tuple[set[str], set[str], set[str]]:
     date_set: set[str] = set()
     ranking_set: set[str] = set()
     attr_set: set[str] = set()
@@ -180,15 +265,15 @@ def _split_and_normalize_filters(viz: CreatedVisualization) -> tuple[set[str], s
     for filter_dict in viz.query.filter_by.values():
         ft = filter_dict.get("type")
         if ft == "date_filter":
-            date_set.add(json.dumps(_normalize_date_filter(filter_dict, fields), sort_keys=True))
+            date_set.add(json.dumps(_normalize_date_filter(filter_dict, fields, today), sort_keys=True))
         elif ft == "ranking_filter":
             ranking_set.add(json.dumps(_normalize_ranking_filter(filter_dict, fields, sole_dim_uri), sort_keys=True))
         elif ft == "attribute_filter":
             attr_set.add(json.dumps(_normalize_attribute_filter(filter_dict, fields), sort_keys=True))
     return date_set, ranking_set, attr_set
 
 
-def normalized_filters(viz: CreatedVisualization) -> dict[str, list[str]]:
+def normalized_filters(viz: CreatedVisualization, today: date | None = None) -> dict[str, list[str]]:
     """A visualization's filters exactly as `check_filters` compares them.
 
     Grouped by the three categories it scores separately and sorted for stable output.
@@ -197,13 +282,18 @@ def normalized_filters(viz: CreatedVisualization) -> dict[str, list[str]]:
     a `filter_date_score` of False otherwise gives no clue whether the period differed,
     the granularity did, or the dataset the filter hangs off did.
     """
-    date_set, ranking_set, attr_set = _split_and_normalize_filters(viz)
+    date_set, ranking_set, attr_set = _split_and_normalize_filters(viz, today)
     return {"date": sorted(date_set), "ranking": sorted(ranking_set), "attribute": sorted(attr_set)}
 
 
-def check_filters(expected: CreatedVisualization, actual: CreatedVisualization) -> FilterScores:
-    exp_date, exp_rank, exp_attr = _split_and_normalize_filters(expected)
-    act_date, act_rank, act_attr = _split_and_normalize_filters(actual)
+def check_filters(
+    expected: CreatedVisualization, actual: CreatedVisualization, today: date | None = None
+) -> FilterScores:
+    # One anchor for both sides: resolving each against its own date.today() would
+    # score inconsistently for a run that straddles midnight.
+    today = today or date.today()
+    exp_date, exp_rank, exp_attr = _split_and_normalize_filters(expected, today)
+    act_date, act_rank, act_attr = _split_and_normalize_filters(actual, today)
     return FilterScores(
         date_ok=act_date == exp_date,
         ranking_ok=act_rank == exp_rank,

packages/gooddata-eval/tests/test_scoring.py

@@ -1,4 +1,7 @@
 # (C) 2026 GoodData Corporation
+from datetime import date
+
+import pytest
 from gooddata_eval.core.models import CreatedVisualization
 from gooddata_eval.core.scoring import (
     check_filters,
@@ -205,3 +208,89 @@ def test_normalized_filters_is_empty_per_category_when_unfiltered():
         }
     )
     assert normalized_filters(viz) == {"date": [], "ranking": [], "attribute": []}
+
+
+# --- relative vs absolute date filters denote the same period ---
+#
+# The agent answers "last month" either relatively (granularity MONTH, from -1, to -1)
+# or absolutely (from 2026-08-01, to 2026-08-31). Compared literally these never match,
+# so a correct answer in whichever encoding the fixture did not happen to use was scored
+# as a wrong date period. Both forms now collapse to the same absolute span.
+
+_TODAY = date(2026, 9, 9)
+
+
+def _date_viz(**overrides):
+    f = {"using": "dataset/dt_transactions", "type": "date_filter"}
+    f.update(overrides)
+    return _viz(query={"fields": {}, "filter_by": {"f_d": f}})
+
+
+def test_check_filters_relative_and_absolute_last_month_agree():
+    expected = _date_viz(**{"from": -1, "to": -1, "granularity": "MONTH"})
+    actual = _date_viz(**{"from": "2026-08-01", "to": "2026-08-31", "granularity": None})
+    assert check_filters(expected, actual, _TODAY).date_ok is True
+
+
+def test_check_filters_absolute_spanning_two_months_still_differs_from_one():
+    """Normalization must not flatten a genuinely wrong window into a match."""
+    expected = _date_viz(**{"from": -1, "to": -1, "granularity": "MONTH"})
+    actual = _date_viz(**{"from": "2026-07-01", "to": "2026-08-31", "granularity": None})
+    assert check_filters(expected, actual, _TODAY).date_ok is False
+
+
+def test_check_filters_day_offsets_resolve_and_off_by_one_still_fails():
+    expected = _date_viz(**{"from": -89, "to": 0, "granularity": "DAY"})
+    assert check_filters(expected, _date_viz(**{"from": "2026-06-12", "to": "2026-09-09"}), _TODAY).date_ok is True
+    assert check_filters(expected, _date_viz(**{"from": -90, "to": 0, "granularity": "DAY"}), _TODAY).date_ok is False
+
+
+def test_check_filters_quarter_and_year_offsets_resolve():
+    q = _date_viz(**{"from": -1, "to": -1, "granularity": "QUARTER"})
+    assert check_filters(q, _date_viz(**{"from": "2026-04-01", "to": "2026-06-30"}), _TODAY).date_ok is True
+    y = _date_viz(**{"from": 0, "to": 0, "granularity": "YEAR"})
+    assert check_filters(y, _date_viz(**{"from": "2026-01-01", "to": "2026-12-31"}), _TODAY).date_ok is True
+
+
+def test_check_filters_week_us_resolves_to_a_sunday_start_week():
+    """WEEK_US is well defined (Sunday-start), unlike a bare WEEK."""
+    expected = _date_viz(**{"from": 0, "to": 0, "granularity": "WEEK_US"})
+    # 2026-09-09 is a Wednesday; its WEEK_US bucket runs Sun 09-06 .. Sat 09-12.
+    assert check_filters(expected, _date_viz(**{"from": "2026-09-06", "to": "2026-09-12"}), _TODAY).date_ok is True
+    assert check_filters(expected, _date_viz(**{"from": "2026-09-06", "to": "2026-09-09"}), _TODAY).date_ok is False
+
+
+def test_check_filters_bare_week_granularity_falls_back_to_literal_comparison():
+    """A bare WEEK is not in the AAC enum and names no convention, so it is not guessed."""
+    expected = _date_viz(**{"from": -2, "to": -1, "granularity": "WEEK"})
+    assert check_filters(expected, _date_viz(**{"from": -2, "to": -1, "granularity": "WEEK"}), _TODAY).date_ok is True
+    assert check_filters(expected, _date_viz(**{"from": -13, "to": 0, "granularity": "DAY"}), _TODAY).date_ok is False
+
+
+def test_check_filters_date_still_distinguishes_the_dataset_it_hangs_off():
+    expected = _date_viz(**{"from": -1, "to": -1, "granularity": "MONTH"})
+    actual = _date_viz(using="dataset/dt_date", **{"from": -1, "to": -1, "granularity": "MONTH"})
+    assert check_filters(expected, actual, _TODAY).date_ok is False
+
+
+@pytest.mark.parametrize(
+    ("granularity", "offset"),
+    [
+        ("YEAR", -_TODAY.year),  # lands on year 0
+        ("DAY", -(10**9)),  # past timedelta's magnitude limit
+        ("WEEK_US", -(10**8)),
+        ("MONTH", -30000),
+        ("QUARTER", -10000),
+    ],
+)
+def test_check_filters_out_of_range_offsets_fall_back_instead_of_raising(granularity, offset):
+    """An offset outside the representable date range must not abort scoring.
+
+    Every granularity can be pushed past date's 1..9999 year range (or timedelta's
+    magnitude limit). Letting the ValueError/OverflowError escape would fail the whole
+    item on one malformed filter, so the span resolves to None and the filter goes back
+    to literal comparison -- which still matches an identically malformed expectation.
+    """
+    expected = _date_viz(**{"from": offset, "to": 0, "granularity": granularity})
+    assert check_filters(expected, _date_viz(**{"from": offset, "to": 0, "granularity": granularity}), _TODAY).date_ok
+    assert not check_filters(expected, _date_viz(**{"from": -1, "to": -1, "granularity": "MONTH"}), _TODAY).date_ok