Merge pull request #1935 from pipecat-ai/aleix/script-evals

improve evals

scripts/evals/README.md

@@ -0,0 +1,55 @@
+# Pipecat Evals
+
+This directory contains a set of utilities to help test Pipecat, specifically
+its examples.
+
+## Release Evals
+
+Before any Pipecat release, we make sure that all (or most) of the examples work
+flawlessly.  We have 100+ examples, and checking each one manually was very
+time-consuming (and painful!), especially because we aim to release often.
+
+To make this process easier, we designed these "release evals," which do the
+following:
+
+- Start one of the foundational examples (the user bot)
+- Start an eval bot
+
+The user bot (i.e. the example) introduces itself, and the eval bot then asks a
+question. The user bot replies, and the eval bot verifies the response.
+
+For example, the eval bot might ask:
+
+"What's 2 plus 2?"
+
+The user bot replies:
+
+"2 plus 2 is 4."
+
+The eval bot (powered by an LLM) evaluates the response and emits a result.  It
+also explains why it thinks the answer is valid or invalid.
+
+To run the release evals:
+
+```sh
+python run-release-evals.py -a -v
+```
+
+This runs all the evals and stores logs and audio (`-a`) for each test.
+
+You can also specify which tests to run. For example, to run all `07` series
+tests:
+
+```sh
+python run-release-evals.py -p 07 -a -v
+```
+
+## Script Evals
+
+You can also run evals for a single example (not part of the release set):
+
+```sh
+python run-eval.py YOUR_EXAMPLE_SCRIPT -a -v
+```
+
+Your script needs to follow any of the foundation examples pattern.

scripts/evals/eval.py

@@ -9,7 +9,6 @@ import asyncio
 import io
 import os
 import re
-import sys
 import time
 import wave
 from datetime import datetime
@@ -17,6 +16,7 @@ from pathlib import Path
 from typing import List, Optional
 
 import aiofiles
+from deepgram import LiveOptions
 from loguru import logger
 from utils import (
     EvalResult,
@@ -45,12 +45,6 @@ from pipecat.transports.services.daily import DailyParams, DailyTransport
 
 SCRIPT_DIR = Path(__file__).resolve().parent
 
-FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational"
-
-sys.path.insert(0, os.path.abspath(FOUNDATIONAL_DIR))
-
-EVAL_PROMPT = ""
-
 PIPELINE_IDLE_TIMEOUT_SECS = 30
 
 
@@ -58,11 +52,13 @@ class EvalRunner:
     def __init__(
         self,
         *,
+        examples_dir: Path,
         pattern: str = "",
         record_audio: bool = False,
         name: Optional[str] = None,
         log_level: str = "DEBUG",
     ):
+        self._examples_dir = examples_dir
         self._pattern = f".*{pattern}.*" if pattern else ""
         self._record_audio = record_audio
         self._log_level = log_level
@@ -79,9 +75,10 @@ class EvalRunner:
         os.makedirs(self._recordings_dir, exist_ok=True)
 
     async def assert_eval(self, params: FunctionCallParams):
+        result = params.arguments["result"]
         reasoning = params.arguments["reasoning"]
-        logger.debug(f"🧠 EVAL REASONING: {reasoning}")
+        logger.debug(f"🧠 EVAL REASONING(result: {result}): {reasoning}")
-        await self._queue.put(params.arguments["result"])
+        await self._queue.put(result)
         await params.result_callback(None)
         await params.llm.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
 
@@ -98,12 +95,14 @@ class EvalRunner:
 
         print_begin_test(example_file)
 
+        script_path = self._examples_dir / example_file
+
         start_time = time.time()
 
         try:
             await asyncio.wait(
                 [
-                    asyncio.create_task(run_example_pipeline(example_file)),
+                    asyncio.create_task(run_example_pipeline(script_path)),
                     asyncio.create_task(run_eval_pipeline(self, example_file, prompt, eval)),
                 ],
                 timeout=90,
@@ -160,11 +159,9 @@ class EvalRunner:
         return os.path.join(self._recordings_dir, f"{base_name}.wav")
 
 
-async def run_example_pipeline(example_file: str):
+async def run_example_pipeline(script_path: Path):
     room_url = os.getenv("DAILY_SAMPLE_ROOM_URL")
 
-    script_path = FOUNDATIONAL_DIR / example_file
-
     module = load_module_from_path(script_path)
 
     transport = DailyTransport(
@@ -199,7 +196,12 @@ async def run_eval_pipeline(
         ),
     )
 
-    stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+    # We disable smart formatting because some times if the user says "3 + 2 is
+    # 5" (in audio) this can be converted to "32 is 5".
+    stt = DeepgramSTTService(
+        api_key=os.getenv("DEEPGRAM_API_KEY"),
+        live_options=LiveOptions(smart_format=False),
+    )
 
     tts = CartesiaTTSService(
         api_key=os.getenv("CARTESIA_API_KEY"),

scripts/evals/run-eval.py

@@ -0,0 +1,50 @@
+#
+# Copyright (c) 2024-2025 Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+from eval import EvalRunner
+from loguru import logger
+from utils import check_env_variables
+
+load_dotenv(override=True)
+
+
+async def main(args: argparse.Namespace):
+    if not check_env_variables():
+        return
+
+    # Log level
+    logger.remove(0)
+    log_level = "TRACE" if args.verbose >= 2 else "DEBUG"
+    if args.verbose:
+        logger.add(sys.stderr, level=log_level)
+
+    script_path = Path(os.path.dirname(os.path.abspath(args.script)))
+    script_file = os.path.basename(args.script)
+
+    runner = EvalRunner(examples_dir=script_path, record_audio=args.audio, log_level=log_level)
+
+    await runner.run_eval(script_file, args.prompt, args.eval)
+
+    runner.print_results()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Pipecat Eval Runner")
+    parser.add_argument("--audio", "-a", action="store_true", help="Record audio for each test")
+    parser.add_argument("--prompt", "-p", required=True, help="Prompt for this eval")
+    parser.add_argument("--eval", "-e", required=False, help="Eval verification")
+    parser.add_argument("--verbose", "-v", action="count", default=0)
+    parser.add_argument("script", help="Script to run")
+    args = parser.parse_args()
+
+    asyncio.run(main(args))

scripts/evals/run-release-evals.py

@@ -8,6 +8,7 @@ import argparse
 import asyncio
 import sys
 from datetime import datetime, timezone
+from pathlib import Path
 
 from dotenv import load_dotenv
 from eval import EvalRunner
@@ -16,6 +17,11 @@ from utils import check_env_variables
 
 load_dotenv(override=True)
 
+SCRIPT_DIR = Path(__file__).resolve().parent
+
+FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational"
+
+
 # Math
 PROMPT_SIMPLE_MATH = "A simple math addition."
 
@@ -124,6 +130,7 @@ async def main(args: argparse.Namespace):
         logger.add(sys.stderr, level=log_level)
 
     runner = EvalRunner(
+        examples_dir=FOUNDATIONAL_DIR,
         name=args.name,
         pattern=args.pattern,
         record_audio=args.audio,