[wip] Add session observability with tags and LLM-based evals

README.md

@@ -18,7 +18,7 @@ The starter project includes:
 - Eval suite based on the LiveKit Agents [testing & evaluation framework](https://docs.livekit.io/agents/start/testing/)
 - [LiveKit Turn Detector](https://docs.livekit.io/agents/logic/turns/turn-detector/), an end-of-turn model that listens to the user's audio directly, combining semantic understanding with acoustic cues for state-of-the-art accuracy across 14 languages
 - [Background voice cancellation](https://docs.livekit.io/transport/media/noise-cancellation/)
-- Deep session insights from LiveKit [Agent Observability](https://docs.livekit.io/deploy/observability/)
+- Deep session insights from LiveKit [Agent Observability](https://docs.livekit.io/deploy/observability/), including [session tags](https://docs.livekit.io/deploy/observability/tags/) and [production evals](https://docs.livekit.io/deploy/observability/evals/)
 - A Dockerfile ready for [production deployment to LiveKit Cloud](https://docs.livekit.io/deploy/agents/)
 
 This starter app is compatible with any [custom web/mobile frontend](https://docs.livekit.io/frontends/) or [telephony](https://docs.livekit.io/telephony/).

src/agent.py

@@ -12,6 +12,12 @@
     inference,
     room_io,
 )
+from livekit.agents.evals import (
+    JudgeGroup,
+    conciseness_judge,
+    relevancy_judge,
+    safety_judge,
+)
 from livekit.plugins import ai_coustics
 
 logger = logging.getLogger("agent")
@@ -91,7 +97,46 @@ def __init__(self) -> None:
 server = AgentServer()
 
 
-@server.rtc_session(agent_name="my-agent")
+async def on_session_end(ctx: JobContext) -> None:
+    # Agent Observability: tag the session so you can find and analyze it in LiveKit Cloud.
+    # Tags and outcomes appear on the session's Insights timeline.
+    # Tags: https://docs.livekit.io/deploy/observability/tags/
+    # Production evals (LLM-as-judge): https://docs.livekit.io/deploy/observability/evals/
+    try:
+        report = ctx.make_session_report()
+    except RuntimeError:
+        # The session never started (for example, the job failed during setup),
+        # so there's nothing to tag.
+        return
+
+    # Mark whether the user actually engaged in a conversation.
+    chat = report.chat_history.copy(
+        exclude_function_call=True, exclude_instructions=True
+    )
+    if len(chat.items) >= 3:
+        ctx.tagger.success()
+    else:
+        ctx.tagger.fail(reason="No meaningful conversation")
+
+    # You can also add your own custom tags with optional metadata, for example:
+    # ctx.tagger.add("turns", metadata={"count": len(chat.items)})
+
+    # Production evals: score every session with LLM-as-judge. Each verdict is
+    # tagged automatically as `lk.judge.<name>`, so you can filter and analyze
+    # sessions by quality in LiveKit Cloud.
+    # https://docs.livekit.io/deploy/observability/evals/
+    #
+    # These built-in judges suit a general assistant; swap in others (accuracy,
+    # coherence, task_completion, tool_use, handoff) or a custom `Judge` subclass
+    # as your agent grows. Each judge adds an LLM call per session.
+    judges = JudgeGroup(
+        llm="openai/gpt-4o-mini",
+        judges=[safety_judge(), relevancy_judge(), conciseness_judge()],
+    )
+    await judges.evaluate(report.chat_history)
+
+
+@server.rtc_session(agent_name="my-agent", on_session_end=on_session_end)
 async def my_agent(ctx: JobContext):
     # Logging setup
     # Add any other context you want in all log entries here