evals: allow running a single eval

scripts/evals/eval.py

@@ -9,7 +9,6 @@ import asyncio
 import io
 import os
 import re
-import sys
 import time
 import wave
 from datetime import datetime
@@ -45,12 +44,6 @@ from pipecat.transports.services.daily import DailyParams, DailyTransport
 
 SCRIPT_DIR = Path(__file__).resolve().parent
 
-FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational"
-
-sys.path.insert(0, os.path.abspath(FOUNDATIONAL_DIR))
-
-EVAL_PROMPT = ""
-
 PIPELINE_IDLE_TIMEOUT_SECS = 30
 
 
@@ -58,11 +51,13 @@ class EvalRunner:
     def __init__(
         self,
         *,
+        examples_dir: Path,
         pattern: str = "",
         record_audio: bool = False,
         name: Optional[str] = None,
         log_level: str = "DEBUG",
     ):
+        self._examples_dir = examples_dir
         self._pattern = f".*{pattern}.*" if pattern else ""
         self._record_audio = record_audio
         self._log_level = log_level
@@ -79,9 +74,10 @@ class EvalRunner:
         os.makedirs(self._recordings_dir, exist_ok=True)
 
     async def assert_eval(self, params: FunctionCallParams):
+        result = params.arguments["result"]
         reasoning = params.arguments["reasoning"]
-        logger.debug(f"🧠 EVAL REASONING: {reasoning}")
-        await self._queue.put(params.arguments["result"])
+        logger.debug(f"🧠 EVAL REASONING(result: {result}): {reasoning}")
+        await self._queue.put(result)
         await params.result_callback(None)
         await params.llm.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
 
@@ -98,12 +94,14 @@ class EvalRunner:
 
         print_begin_test(example_file)
 
+        script_path = self._examples_dir / example_file
+
         start_time = time.time()
 
         try:
             await asyncio.wait(
                 [
-                    asyncio.create_task(run_example_pipeline(example_file)),
+                    asyncio.create_task(run_example_pipeline(script_path)),
                     asyncio.create_task(run_eval_pipeline(self, example_file, prompt, eval)),
                 ],
                 timeout=90,
@@ -160,11 +158,9 @@ class EvalRunner:
         return os.path.join(self._recordings_dir, f"{base_name}.wav")
 
 
-async def run_example_pipeline(example_file: str):
+async def run_example_pipeline(script_path: Path):
     room_url = os.getenv("DAILY_SAMPLE_ROOM_URL")
 
-    script_path = FOUNDATIONAL_DIR / example_file
-
     module = load_module_from_path(script_path)
 
     transport = DailyTransport(