evals: allow running a single eval

scripts/evals/README.md

@@ -43,3 +43,13 @@ tests:
 ```sh
 python run-release-evals.py -p 07 -a -v
 ```
+
+## Script Evals
+
+You can also run evals for a single example (not part of the release set):
+
+```sh
+python run-eval.py YOUR_EXAMPLE_SCRIPT -a -v
+```
+
+Your script needs to follow any of the foundation examples pattern.

scripts/evals/eval.py

@@ -9,7 +9,6 @@ import asyncio
 import io
 import os
 import re
-import sys
 import time
 import wave
 from datetime import datetime
@@ -45,12 +44,6 @@ from pipecat.transports.services.daily import DailyParams, DailyTransport
 
 SCRIPT_DIR = Path(__file__).resolve().parent
 
-FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational"
-
-sys.path.insert(0, os.path.abspath(FOUNDATIONAL_DIR))
-
-EVAL_PROMPT = ""
-
 PIPELINE_IDLE_TIMEOUT_SECS = 30
 
 
@@ -58,11 +51,13 @@ class EvalRunner:
     def __init__(
         self,
         *,
+        examples_dir: Path,
         pattern: str = "",
         record_audio: bool = False,
         name: Optional[str] = None,
         log_level: str = "DEBUG",
     ):
+        self._examples_dir = examples_dir
         self._pattern = f".*{pattern}.*" if pattern else ""
         self._record_audio = record_audio
         self._log_level = log_level
@@ -79,9 +74,10 @@ class EvalRunner:
         os.makedirs(self._recordings_dir, exist_ok=True)
 
     async def assert_eval(self, params: FunctionCallParams):
+        result = params.arguments["result"]
         reasoning = params.arguments["reasoning"]
-        logger.debug(f"🧠 EVAL REASONING: {reasoning}")
+        logger.debug(f"🧠 EVAL REASONING(result: {result}): {reasoning}")
-        await self._queue.put(params.arguments["result"])
+        await self._queue.put(result)
         await params.result_callback(None)
         await params.llm.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
 
@@ -98,12 +94,14 @@ class EvalRunner:
 
         print_begin_test(example_file)
 
+        script_path = self._examples_dir / example_file
+
         start_time = time.time()
 
         try:
             await asyncio.wait(
                 [
-                    asyncio.create_task(run_example_pipeline(example_file)),
+                    asyncio.create_task(run_example_pipeline(script_path)),
                     asyncio.create_task(run_eval_pipeline(self, example_file, prompt, eval)),
                 ],
                 timeout=90,
@@ -160,11 +158,9 @@ class EvalRunner:
         return os.path.join(self._recordings_dir, f"{base_name}.wav")
 
 
-async def run_example_pipeline(example_file: str):
+async def run_example_pipeline(script_path: Path):
     room_url = os.getenv("DAILY_SAMPLE_ROOM_URL")
 
-    script_path = FOUNDATIONAL_DIR / example_file
-
     module = load_module_from_path(script_path)
 
     transport = DailyTransport(

scripts/evals/run-eval.py

@@ -0,0 +1,50 @@
+#
+# Copyright (c) 2024-2025 Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+from eval import EvalRunner
+from loguru import logger
+from utils import check_env_variables
+
+load_dotenv(override=True)
+
+
+async def main(args: argparse.Namespace):
+    if not check_env_variables():
+        return
+
+    # Log level
+    logger.remove(0)
+    log_level = "TRACE" if args.verbose >= 2 else "DEBUG"
+    if args.verbose:
+        logger.add(sys.stderr, level=log_level)
+
+    script_path = Path(os.path.dirname(os.path.abspath(args.script)))
+    script_file = os.path.basename(args.script)
+
+    runner = EvalRunner(examples_dir=script_path, record_audio=args.audio, log_level=log_level)
+
+    await runner.run_eval(script_file, args.prompt, args.eval)
+
+    runner.print_results()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Pipecat Eval Runner")
+    parser.add_argument("--audio", "-a", action="store_true", help="Record audio for each test")
+    parser.add_argument("--prompt", "-p", required=True, help="Prompt for this eval")
+    parser.add_argument("--eval", "-e", required=False, help="Eval verification")
+    parser.add_argument("--verbose", "-v", action="count", default=0)
+    parser.add_argument("script", help="Script to run")
+    args = parser.parse_args()
+
+    asyncio.run(main(args))

scripts/evals/run-release-evals.py

@@ -8,6 +8,7 @@ import argparse
 import asyncio
 import sys
 from datetime import datetime, timezone
+from pathlib import Path
 
 from dotenv import load_dotenv
 from eval import EvalRunner
@@ -16,6 +17,11 @@ from utils import check_env_variables
 
 load_dotenv(override=True)
 
+SCRIPT_DIR = Path(__file__).resolve().parent
+
+FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational"
+
+
 # Math
 PROMPT_SIMPLE_MATH = "A simple math addition."
 
@@ -124,6 +130,7 @@ async def main(args: argparse.Namespace):
         logger.add(sys.stderr, level=log_level)
 
     runner = EvalRunner(
+        examples_dir=FOUNDATIONAL_DIR,
         name=args.name,
         pattern=args.pattern,
         record_audio=args.audio,