From 7b4a5d622ec21c44da5ac327123a68d28dd2ab0c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 22:38:46 +0000 Subject: [PATCH 1/3] Add session tagging and eval pointers to starter agent Wire an on_session_end callback that tags the session outcome via ctx.tagger and points to the Agent Observability tags and evals docs. https://claude.ai/code/session_01Mkt7G86CNXSTmhRrwHcL2A --- README.md | 2 +- src/agent.py | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bea2f7d..1787609 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The starter project includes: - Eval suite based on the LiveKit Agents [testing & evaluation framework](https://docs.livekit.io/agents/start/testing/) - [LiveKit Turn Detector](https://docs.livekit.io/agents/logic/turns/turn-detector/) for contextually-aware speaker detection, with multilingual support - [Background voice cancellation](https://docs.livekit.io/transport/media/noise-cancellation/) -- Deep session insights from LiveKit [Agent Observability](https://docs.livekit.io/deploy/observability/) +- Deep session insights from LiveKit [Agent Observability](https://docs.livekit.io/deploy/observability/), including [session tags](https://docs.livekit.io/deploy/observability/tags/) and [production evals](https://docs.livekit.io/deploy/observability/evals/) - A Dockerfile ready for [production deployment to LiveKit Cloud](https://docs.livekit.io/deploy/agents/) This starter app is compatible with any [custom web/mobile frontend](https://docs.livekit.io/frontends/) or [telephony](https://docs.livekit.io/telephony/). diff --git a/src/agent.py b/src/agent.py index 1076905..518e3fe 100644 --- a/src/agent.py +++ b/src/agent.py @@ -99,7 +99,32 @@ def prewarm(proc: JobProcess): server.setup_fnc = prewarm -@server.rtc_session(agent_name="my-agent") +async def on_session_end(ctx: JobContext) -> None: + # Agent Observability: tag the session so you can find and analyze it in LiveKit Cloud. + # Tags and outcomes appear on the session's Insights timeline. + # Tags: https://docs.livekit.io/deploy/observability/tags/ + # Production evals (LLM-as-judge): https://docs.livekit.io/deploy/observability/evals/ + try: + report = ctx.make_session_report() + except RuntimeError: + # The session never started (for example, the job failed during setup), + # so there's nothing to tag. + return + + # Mark whether the user actually engaged in a conversation. + chat = report.chat_history.copy( + exclude_function_call=True, exclude_instructions=True + ) + if len(chat.items) >= 3: + ctx.tagger.success() + else: + ctx.tagger.fail(reason="No meaningful conversation") + + # You can also add your own custom tags with optional metadata, for example: + # ctx.tagger.add("turns", metadata={"count": len(chat.items)}) + + +@server.rtc_session(agent_name="my-agent", on_session_end=on_session_end) async def my_agent(ctx: JobContext): # Logging setup # Add any other context you want in all log entries here From f2a04c2a895d8e5c2927cd3ea79277d789e3a4df Mon Sep 17 00:00:00 2001 From: Ben Cherry Date: Mon, 10 Aug 2026 13:59:48 -0700 Subject: [PATCH 2/3] Add LLM-as-judge production evals to session-end example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The on_session_end handler previously demonstrated only heuristic outcome tagging (chat length). Add a JudgeGroup running safety, coherence, and conciseness judges to also demonstrate production evals — verdicts are auto-tagged as lk.judge.. Judging is gated behind the engagement check so empty sessions don't incur a model call. --- src/agent.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/agent.py b/src/agent.py index 9266727..2fee63f 100644 --- a/src/agent.py +++ b/src/agent.py @@ -12,6 +12,12 @@ inference, room_io, ) +from livekit.agents.evals import ( + JudgeGroup, + coherence_judge, + conciseness_judge, + safety_judge, +) from livekit.plugins import ai_coustics logger = logging.getLogger("agent") @@ -103,18 +109,35 @@ async def on_session_end(ctx: JobContext) -> None: # so there's nothing to tag. return - # Mark whether the user actually engaged in a conversation. + # Mark whether the user actually engaged in a conversation. This is a cheap, + # instant heuristic — no model call — so it runs on every session. chat = report.chat_history.copy( exclude_function_call=True, exclude_instructions=True ) - if len(chat.items) >= 3: - ctx.tagger.success() - else: + if len(chat.items) < 3: ctx.tagger.fail(reason="No meaningful conversation") + return + + ctx.tagger.success() # You can also add your own custom tags with optional metadata, for example: # ctx.tagger.add("turns", metadata={"count": len(chat.items)}) + # Production evals: score the conversation with LLM-as-judge. Each verdict is + # tagged automatically as `lk.judge.`, so you can filter and analyze + # sessions by quality in LiveKit Cloud. + # https://docs.livekit.io/deploy/observability/evals/ + # + # Judging runs an LLM per session, so it's gated behind the engagement check + # above — empty or abandoned sessions don't incur a model call. These built-in + # judges suit a general assistant; swap in others (accuracy, task_completion, + # tool_use, handoff) or a custom `Judge` subclass as your agent grows. + judges = JudgeGroup( + llm="openai/gpt-4o-mini", + judges=[safety_judge(), coherence_judge(), conciseness_judge()], + ) + await judges.evaluate(report.chat_history) + @server.rtc_session(agent_name="my-agent", on_session_end=on_session_end) async def my_agent(ctx: JobContext): From 59d1fd6648ecb54a6388dfe78cab10f980f69a08 Mon Sep 17 00:00:00 2001 From: Ben Cherry Date: Mon, 10 Aug 2026 14:03:06 -0700 Subject: [PATCH 3/3] Run production evals on every session; use relevancy judge Per review: judges should run unconditionally (not gated behind the engagement heuristic), and relevancy fits a general assistant better than coherence. Swap coherence_judge -> relevancy_judge and remove the early return so the JudgeGroup evaluates every session. Outcome tagging (success/fail) is unchanged. --- src/agent.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/agent.py b/src/agent.py index 2fee63f..6deb3d6 100644 --- a/src/agent.py +++ b/src/agent.py @@ -14,8 +14,8 @@ ) from livekit.agents.evals import ( JudgeGroup, - coherence_judge, conciseness_judge, + relevancy_judge, safety_judge, ) from livekit.plugins import ai_coustics @@ -109,32 +109,29 @@ async def on_session_end(ctx: JobContext) -> None: # so there's nothing to tag. return - # Mark whether the user actually engaged in a conversation. This is a cheap, - # instant heuristic — no model call — so it runs on every session. + # Mark whether the user actually engaged in a conversation. chat = report.chat_history.copy( exclude_function_call=True, exclude_instructions=True ) - if len(chat.items) < 3: + if len(chat.items) >= 3: + ctx.tagger.success() + else: ctx.tagger.fail(reason="No meaningful conversation") - return - - ctx.tagger.success() # You can also add your own custom tags with optional metadata, for example: # ctx.tagger.add("turns", metadata={"count": len(chat.items)}) - # Production evals: score the conversation with LLM-as-judge. Each verdict is + # Production evals: score every session with LLM-as-judge. Each verdict is # tagged automatically as `lk.judge.`, so you can filter and analyze # sessions by quality in LiveKit Cloud. # https://docs.livekit.io/deploy/observability/evals/ # - # Judging runs an LLM per session, so it's gated behind the engagement check - # above — empty or abandoned sessions don't incur a model call. These built-in - # judges suit a general assistant; swap in others (accuracy, task_completion, - # tool_use, handoff) or a custom `Judge` subclass as your agent grows. + # These built-in judges suit a general assistant; swap in others (accuracy, + # coherence, task_completion, tool_use, handoff) or a custom `Judge` subclass + # as your agent grows. Each judge adds an LLM call per session. judges = JudgeGroup( llm="openai/gpt-4o-mini", - judges=[safety_judge(), coherence_judge(), conciseness_judge()], + judges=[safety_judge(), relevancy_judge(), conciseness_judge()], ) await judges.evaluate(report.chat_history)