diff --git a/README.md b/README.md index eb0f4fd..5c474da 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ The starter project includes: - Eval suite based on the LiveKit Agents [testing & evaluation framework](https://docs.livekit.io/agents/start/testing/) - [LiveKit Turn Detector](https://docs.livekit.io/agents/logic/turns/turn-detector/), an end-of-turn model that listens to the user's audio directly, combining semantic understanding with acoustic cues for state-of-the-art accuracy across 14 languages - [Background voice cancellation](https://docs.livekit.io/transport/media/noise-cancellation/) -- Deep session insights from LiveKit [Agent Observability](https://docs.livekit.io/deploy/observability/) +- Deep session insights from LiveKit [Agent Observability](https://docs.livekit.io/deploy/observability/), including [session tags](https://docs.livekit.io/deploy/observability/tags/) and [production evals](https://docs.livekit.io/deploy/observability/evals/) - A Dockerfile ready for [production deployment to LiveKit Cloud](https://docs.livekit.io/deploy/agents/) This starter app is compatible with any [custom web/mobile frontend](https://docs.livekit.io/frontends/) or [telephony](https://docs.livekit.io/telephony/). diff --git a/src/agent.py b/src/agent.py index 17e6a91..75f7f2a 100644 --- a/src/agent.py +++ b/src/agent.py @@ -12,6 +12,12 @@ inference, room_io, ) +from livekit.agents.evals import ( + JudgeGroup, + conciseness_judge, + relevancy_judge, + safety_judge, +) from livekit.plugins import ai_coustics logger = logging.getLogger("agent") @@ -91,7 +97,46 @@ def __init__(self) -> None: server = AgentServer() -@server.rtc_session(agent_name="my-agent") +async def on_session_end(ctx: JobContext) -> None: + # Agent Observability: tag the session so you can find and analyze it in LiveKit Cloud. + # Tags and outcomes appear on the session's Insights timeline. + # Tags: https://docs.livekit.io/deploy/observability/tags/ + # Production evals (LLM-as-judge): https://docs.livekit.io/deploy/observability/evals/ + try: + report = ctx.make_session_report() + except RuntimeError: + # The session never started (for example, the job failed during setup), + # so there's nothing to tag. + return + + # Mark whether the user actually engaged in a conversation. + chat = report.chat_history.copy( + exclude_function_call=True, exclude_instructions=True + ) + if len(chat.items) >= 3: + ctx.tagger.success() + else: + ctx.tagger.fail(reason="No meaningful conversation") + + # You can also add your own custom tags with optional metadata, for example: + # ctx.tagger.add("turns", metadata={"count": len(chat.items)}) + + # Production evals: score every session with LLM-as-judge. Each verdict is + # tagged automatically as `lk.judge.`, so you can filter and analyze + # sessions by quality in LiveKit Cloud. + # https://docs.livekit.io/deploy/observability/evals/ + # + # These built-in judges suit a general assistant; swap in others (accuracy, + # coherence, task_completion, tool_use, handoff) or a custom `Judge` subclass + # as your agent grows. Each judge adds an LLM call per session. + judges = JudgeGroup( + llm="openai/gpt-4o-mini", + judges=[safety_judge(), relevancy_judge(), conciseness_judge()], + ) + await judges.evaluate(report.chat_history) + + +@server.rtc_session(agent_name="my-agent", on_session_end=on_session_end) async def my_agent(ctx: JobContext): # Logging setup # Add any other context you want in all log entries here