Skip to content

feat(gooddata-eval): record why an agentic simulated-user loop stopped #1789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Tomkess wants to merge 3 commits into
base: master
Choose a base branch
from
Open

feat(gooddata-eval): record why an agentic simulated-user loop stopped #1789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c9a9100
feat(gooddata-eval): record why an agentic simulated-user loop stopped
Tomkess Sep 9, 2026
5c819b1
fix(gooddata-eval): address CodeRabbit review on the loop exit_reason…
Tomkess Sep 9, 2026
d0eb73a
fix(gooddata-eval): record chat and simulated-user faults instead of …
Tomkess Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 43 additions & 5 deletions packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@
utc_now,
)
from gooddata_eval.core.chat.render import render_answer_text
from gooddata_eval.core.chat.sse_client import ChatClient
from gooddata_eval.core.chat.sse_client import ChatClient, ChatError
from gooddata_eval.core.config import ReasoningEffort
from gooddata_eval.core.models import (
AgenticAssertionError,
AgenticEvalOutcome,
LoopExit,
ReasoningStepEvent,
ToolCallEvent,
build_latency_breakdown,
Expand Down Expand Up @@ -473,6 +474,12 @@ class AlertRunResult:
response_id: str | None = None
tool_call_events: list[ToolCallEvent] = field(default_factory=list)
reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
# Why the simulated-user loop stopped, and how many turns it took. Without these a run
# that ran out of turns is indistinguishable from one that refused: both land on
# alert_created=False, and every downstream check is `alert_created and ...`, so both
# also report operator/threshold/metric/recipients as False.
exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED
turns_used: int = 0


@dataclass
Expand Down Expand Up @@ -669,8 +676,20 @@ def _run_once(conv_id: str) -> AlertRunResult:
conversation_history: list = []
current_question = question

# Defaults to BUDGET_EXHAUSTED: every other exit sets it explicitly, so a loop
# that simply runs out of range() is correctly labelled without a trailing else.
exit_reason = LoopExit.BUDGET_EXHAUSTED
turns_used = 0
for _iteration in range(max_iterations):
chat_result = client.send_message(conv_id, current_question)
turns_used = _iteration + 1
try:
chat_result = client.send_message(conv_id, current_question)
except ChatError as exc:
# Without this the exception escapes run_agentic_alert_skill entirely,
# discarding every K-run already completed along with any exit_reason.
print(f"[CHAT] send_message failed for conversation {conv_id}: {exc}")
exit_reason = LoopExit.CHAT_ERROR
break
reasoning_steps.extend(chat_result.reasoning_steps or [])
response_id = chat_result.response_id or response_id
turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
Expand All @@ -684,6 +703,7 @@ def _run_once(conv_id: str) -> AlertRunResult:
alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
if tool_called:
alert_id_to_delete = alert_id
exit_reason = LoopExit.SUCCESS
break
response_text = (chat_result.text_response or "").strip()
if not response_text and chat_result.alert_proposals:
Expand All @@ -692,13 +712,24 @@ def _run_once(conv_id: str) -> AlertRunResult:
response_text = render_answer_text(chat_result)
# Stop if agent gave a completely empty response (stuck)
if not response_text and not chat_result.tool_call_events:
exit_reason = LoopExit.AGENT_SILENT
break
# Stop before generating a follow-up for the last iteration
if _iteration >= max_iterations - 1:
break
follow_up = generate_simulated_alert_response(
response_text, expected, conversation_history, question=question
)
# Recorded rather than raised, matching metric_skill and kda_skill. Letting it
# propagate did keep a harness fault from being scored as a content failure,
# but it also discarded the K-runs already completed -- and SIMULATED_USER_FAILED
# achieves the same separation while keeping them, since reporting reads the
# exit reason to classify the run as an error rather than an agent failure.
try:
follow_up = generate_simulated_alert_response(
response_text, expected, conversation_history, question=question
)
except Exception as exc: # noqa: BLE001 -- harness-side fault; end only this run
print(f"[SIM-USER] Simulated reply failed for conversation {conv_id}: {exc}")
exit_reason = LoopExit.SIMULATED_USER_FAILED
break
# Record this exchange so the next call has full history
conversation_history.append({"role": "assistant", "content": response_text})
conversation_history.append({"role": "user", "content": follow_up})
Expand All @@ -724,6 +755,8 @@ def _run_once(conv_id: str) -> AlertRunResult:
response_id=response_id,
tool_call_events=all_tool_call_events,
reasoning_step_events=all_reasoning_step_events,
exit_reason=exit_reason,
turns_used=turns_used,
)
finally:
if alert_id_to_delete:
Expand Down Expand Up @@ -886,6 +919,11 @@ def _write_scores(ctx: RunTraceContext) -> None:
"attributes_correct": ev.attributes_correct,
"granularity_correct": ev.granularity_correct,
"actual_alert_arguments": best.actual_alert_arguments,
# Why the loop stopped. alert_created=False alone cannot tell a refusal from a run
# that hit max_iterations while still on track -- see LoopExit.
"exit_reason": best.exit_reason.value,
"turns_used": best.turns_used,
"max_iterations": max_iterations,
"latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
}

Expand Down
50 changes: 47 additions & 3 deletions packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,13 @@
from gooddata_eval.core.agentic.alert_skill import render_alert_proposal
from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids, _extract_metric_result
from gooddata_eval.core.chat.render import render_answer_text
from gooddata_eval.core.chat.sse_client import ChatClient
from gooddata_eval.core.chat.sse_client import ChatClient, ChatError
from gooddata_eval.core.config import ReasoningEffort
from gooddata_eval.core.models import (
AgenticAssertionError,
AgenticEvalOutcome,
ChatResult,
LoopExit,
ReasoningStepEvent,
ToolCallEvent,
build_latency_breakdown,
Expand Down Expand Up @@ -93,6 +94,10 @@ class TurnResult(BaseModel):
active_skills: list[str] = Field(default_factory=list)
clarification_turns_used: int = 0
output_correct: bool | None = None
# Why this turn's clarification loop stopped -- see LoopExit. output_present=False alone
# cannot separate a turn that ran out of clarification budget from one where the agent
# went silent, and skill_success folds both into the same failure.
exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED

@property
def skill_success(self) -> bool:
Expand All @@ -112,6 +117,10 @@ def skill_success(self) -> bool:
# What skill_routing was judged against -- without it, a turn showing
# skill_routing=True and activated_skills=[] looks like a scoring bug.
"active_skills",
# Why the clarification loop ended on this turn, and how much of the budget it
# took to get there -- exit_reason alone cannot be related to the limit without it.
"exit_reason",
"clarification_turns_used",
}

def detail(self) -> dict:
Expand Down Expand Up @@ -336,6 +345,10 @@ class ConversationResult:
full_skill_coverage: bool
conversation_success: bool
total_clarification_turns: int
# The configured per-turn clarification budget, so a reader can tell a turn that used
# its whole allowance from one that stopped early. Every other agentic kind reports its
# limit in detail; without this, conversation is the exception to that contract.
max_clarification_turns: int = _DEFAULT_MAX_CLARIFICATION_TURNS
reasoning_steps: list[str] = field(default_factory=list)
response_id: str | None = None
tool_call_events: list[ToolCallEvent] = field(default_factory=list)
Expand Down Expand Up @@ -421,6 +434,10 @@ def run_agentic_conversation(
# read as "nothing was active", which is a different claim.
active_skills=sorted(active_skills),
output_correct=False,
# This turn's loop never ran at all -- a $ref pointing at an earlier
# turn's output could not be resolved. Labelling it BUDGET_EXHAUSTED
# (the field default) would claim it ran out of clarification turns.
exit_reason=LoopExit.NOT_RUN,
)
)
continue
Expand All @@ -431,8 +448,27 @@ def run_agentic_conversation(
current_message = turn.message
final_result: ChatResult | None = None

# Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop
# that simply runs out of range() is labelled correctly with no trailing else.
turn_exit = LoopExit.BUDGET_EXHAUSTED
for _iter in range(max_clarification_turns + 1):
chat_result = client.send_message(conversation_id, current_message)
try:
chat_result = client.send_message(conversation_id, current_message)
except ChatError as exc:
# Recorded rather than raised so the turns already completed keep their
# results, and so this turn is distinguishable from one where the agent
# simply failed to produce output. no_error below reads this back.
print(f"[CHAT] send_message failed for conversation {conversation_id}: {exc}")
partial = getattr(exc, "partial_result", None)
if partial is not None:
final_result = partial
all_tool_calls.extend(partial.tool_call_events or [])
conversation_tool_call_events.extend(partial.tool_call_events or [])
conversation_reasoning_step_events.extend(partial.reasoning_step_events or [])
reasoning_steps.extend(partial.reasoning_steps or [])
response_id = partial.response_id or response_id
turn_exit = LoopExit.CHAT_ERROR
break
final_result = chat_result
turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
chat_result,
Expand All @@ -447,6 +483,7 @@ def run_agentic_conversation(
response_id = chat_result.response_id or response_id

if _check_output_present(resolved_turn, chat_result):
turn_exit = LoopExit.SUCCESS
break

response_text = (chat_result.text_response or "").strip()
Expand All @@ -455,6 +492,7 @@ def run_agentic_conversation(
if not response_text:
response_text = render_answer_text(chat_result)
if not response_text and not chat_result.tool_call_events:
turn_exit = LoopExit.AGENT_SILENT
break
if clarification_turns >= max_clarification_turns:
break
Expand Down Expand Up @@ -495,11 +533,15 @@ def run_agentic_conversation(
expected_skill=turn.expected_skill,
skill_routing=skill_routing,
output_present=output_present,
no_error=True, # SDK raises on errors; reaching here means no critical error.
# A chat fault used to escape the whole run, so reaching here did mean no
# error. Now that it is caught and recorded, this has to read it back --
# otherwise a turn whose chat call failed reports no_error=True.
no_error=turn_exit is not LoopExit.CHAT_ERROR,
activated_skills=declared or [],
active_skills=sorted(active_skills),
clarification_turns_used=clarification_turns,
output_correct=output_correct,
exit_reason=turn_exit,
Comment thread
coderabbitai[bot] marked this conversation as resolved.
)
)

Expand All @@ -522,6 +564,7 @@ def run_agentic_conversation(
full_skill_coverage=full_skill_coverage,
conversation_success=conversation_success,
total_clarification_turns=total_clarification_turns,
max_clarification_turns=max_clarification_turns,
reasoning_steps=reasoning_steps,
response_id=response_id,
tool_call_events=conversation_tool_call_events,
Expand All @@ -533,6 +576,7 @@ def _conversation_detail(result: ConversationResult) -> dict:
return {
"full_skill_coverage": result.full_skill_coverage,
"total_clarification_turns": result.total_clarification_turns,
"max_clarification_turns": result.max_clarification_turns,
"turns": [tr.detail() for tr in result.turn_results],
"latency_breakdown": build_latency_breakdown(result.tool_call_events, result.reasoning_step_events),
}
Expand Down
20 changes: 20 additions & 0 deletions packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
AgenticAssertionError,
AgenticEvalOutcome,
ChatResult,
LoopExit,
ReasoningStepEvent,
ToolCallEvent,
build_latency_breakdown,
Expand Down Expand Up @@ -181,6 +182,10 @@ class KdaRunResult:
response_id: str | None = None
tool_call_events: list[ToolCallEvent] = field(default_factory=list)
reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
# Why the simulated-user loop stopped -- see LoopExit. `triggered=False` alone cannot
# separate a refusal from a run that hit max_iterations while still on track.
exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED
turns_used: int = 0


@dataclass
Expand Down Expand Up @@ -268,7 +273,12 @@ def _accumulate(result: ChatResult) -> None:
all_tool_call_events.extend(result.tool_call_events or [])
all_reasoning_step_events.extend(result.reasoning_step_events or [])

# Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop that
# simply runs out of range() is labelled correctly with no trailing else.
exit_reason = LoopExit.BUDGET_EXHAUSTED
turns_used = 0
for iteration in range(max_iterations):
turns_used = iteration + 1
try:
chat_result = client.send_message(conv_id, current_question)
except Exception as exc: # noqa: BLE001 -- end this run, not the whole assertion
Expand All @@ -282,6 +292,7 @@ def _accumulate(result: ChatResult) -> None:
if create_args is not None:
turn_wall_clock_sec = partial.turn_wall_clock_sec
turn_completed = False
exit_reason = LoopExit.CHAT_ERROR
break
reasoning_steps.extend(chat_result.reasoning_steps or [])
response_id = chat_result.response_id or response_id
Expand All @@ -296,8 +307,10 @@ def _accumulate(result: ChatResult) -> None:
# final either way -- execute_result may still be None (e.g. the skill's
# execute tool isn't available at all when data-sharing is off for the org).
turn_wall_clock_sec = chat_result.turn_wall_clock_sec
exit_reason = LoopExit.SUCCESS
break
if not response_text:
exit_reason = LoopExit.AGENT_SILENT
break
if iteration >= max_iterations - 1:
break
Expand All @@ -313,6 +326,7 @@ def _accumulate(result: ChatResult) -> None:
disambiguated = True
except Exception as exc: # noqa: BLE001 -- safety net, not the assertion; end only this run
_log.warning("Simulated KDA user reply failed for conversation %s: %s", conv_id, exc)
exit_reason = LoopExit.SIMULATED_USER_FAILED
break

ev = _evaluate_run(create_args, execute_result, turn_completed, disambiguated)
Expand All @@ -326,6 +340,8 @@ def _accumulate(result: ChatResult) -> None:
response_id=response_id,
tool_call_events=all_tool_call_events,
reasoning_step_events=all_reasoning_step_events,
exit_reason=exit_reason,
turns_used=turns_used,
)

try:
Expand Down Expand Up @@ -482,6 +498,10 @@ def _write_scores(ctx: RunTraceContext) -> None:
"disambiguated": ev.disambiguated,
"actual_create_args": best.actual_create_args,
"actual_execute_result": best.actual_execute_result,
# Why the loop stopped -- see LoopExit.
"exit_reason": best.exit_reason.value,
"turns_used": best.turns_used,
"max_iterations": max_iterations,
"latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
}

Expand Down
Loading
Loading