From f1df0795125ba6b7e20f9ed89f69b011fca17887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 30 May 2025 15:41:13 -0700 Subject: [PATCH] evals: allow running a single eval --- scripts/evals/README.md | 10 ++++++ scripts/evals/eval.py | 22 ++++++------- scripts/evals/run-eval.py | 50 ++++++++++++++++++++++++++++++ scripts/evals/run-release-evals.py | 7 +++++ 4 files changed, 76 insertions(+), 13 deletions(-) create mode 100644 scripts/evals/run-eval.py diff --git a/scripts/evals/README.md b/scripts/evals/README.md index d22102f4b..ad94e4605 100644 --- a/scripts/evals/README.md +++ b/scripts/evals/README.md @@ -43,3 +43,13 @@ tests: ```sh python run-release-evals.py -p 07 -a -v ``` + +## Script Evals + +You can also run evals for a single example (not part of the release set): + +```sh +python run-eval.py YOUR_EXAMPLE_SCRIPT -a -v +``` + +Your script needs to follow any of the foundation examples pattern. diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index 5ad2bb8f3..47959de3f 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -9,7 +9,6 @@ import asyncio import io import os import re -import sys import time import wave from datetime import datetime @@ -45,12 +44,6 @@ from pipecat.transports.services.daily import DailyParams, DailyTransport SCRIPT_DIR = Path(__file__).resolve().parent -FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational" - -sys.path.insert(0, os.path.abspath(FOUNDATIONAL_DIR)) - -EVAL_PROMPT = "" - PIPELINE_IDLE_TIMEOUT_SECS = 30 @@ -58,11 +51,13 @@ class EvalRunner: def __init__( self, *, + examples_dir: Path, pattern: str = "", record_audio: bool = False, name: Optional[str] = None, log_level: str = "DEBUG", ): + self._examples_dir = examples_dir self._pattern = f".*{pattern}.*" if pattern else "" self._record_audio = record_audio self._log_level = log_level @@ -79,9 +74,10 @@ class EvalRunner: os.makedirs(self._recordings_dir, exist_ok=True) async def assert_eval(self, params: FunctionCallParams): + result = params.arguments["result"] reasoning = params.arguments["reasoning"] - logger.debug(f"🧠 EVAL REASONING: {reasoning}") - await self._queue.put(params.arguments["result"]) + logger.debug(f"🧠 EVAL REASONING(result: {result}): {reasoning}") + await self._queue.put(result) await params.result_callback(None) await params.llm.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) @@ -98,12 +94,14 @@ class EvalRunner: print_begin_test(example_file) + script_path = self._examples_dir / example_file + start_time = time.time() try: await asyncio.wait( [ - asyncio.create_task(run_example_pipeline(example_file)), + asyncio.create_task(run_example_pipeline(script_path)), asyncio.create_task(run_eval_pipeline(self, example_file, prompt, eval)), ], timeout=90, @@ -160,11 +158,9 @@ class EvalRunner: return os.path.join(self._recordings_dir, f"{base_name}.wav") -async def run_example_pipeline(example_file: str): +async def run_example_pipeline(script_path: Path): room_url = os.getenv("DAILY_SAMPLE_ROOM_URL") - script_path = FOUNDATIONAL_DIR / example_file - module = load_module_from_path(script_path) transport = DailyTransport( diff --git a/scripts/evals/run-eval.py b/scripts/evals/run-eval.py new file mode 100644 index 000000000..4ea4aa56b --- /dev/null +++ b/scripts/evals/run-eval.py @@ -0,0 +1,50 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import asyncio +import os +import sys +from pathlib import Path + +from dotenv import load_dotenv +from eval import EvalRunner +from loguru import logger +from utils import check_env_variables + +load_dotenv(override=True) + + +async def main(args: argparse.Namespace): + if not check_env_variables(): + return + + # Log level + logger.remove(0) + log_level = "TRACE" if args.verbose >= 2 else "DEBUG" + if args.verbose: + logger.add(sys.stderr, level=log_level) + + script_path = Path(os.path.dirname(os.path.abspath(args.script))) + script_file = os.path.basename(args.script) + + runner = EvalRunner(examples_dir=script_path, record_audio=args.audio, log_level=log_level) + + await runner.run_eval(script_file, args.prompt, args.eval) + + runner.print_results() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Pipecat Eval Runner") + parser.add_argument("--audio", "-a", action="store_true", help="Record audio for each test") + parser.add_argument("--prompt", "-p", required=True, help="Prompt for this eval") + parser.add_argument("--eval", "-e", required=False, help="Eval verification") + parser.add_argument("--verbose", "-v", action="count", default=0) + parser.add_argument("script", help="Script to run") + args = parser.parse_args() + + asyncio.run(main(args)) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 33511fdeb..242dd6fc5 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -8,6 +8,7 @@ import argparse import asyncio import sys from datetime import datetime, timezone +from pathlib import Path from dotenv import load_dotenv from eval import EvalRunner @@ -16,6 +17,11 @@ from utils import check_env_variables load_dotenv(override=True) +SCRIPT_DIR = Path(__file__).resolve().parent + +FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational" + + # Math PROMPT_SIMPLE_MATH = "A simple math addition." @@ -124,6 +130,7 @@ async def main(args: argparse.Namespace): logger.add(sys.stderr, level=log_level) runner = EvalRunner( + examples_dir=FOUNDATIONAL_DIR, name=args.name, pattern=args.pattern, record_audio=args.audio,