From 46deaada70a0ca016c33759f64ab918f571742a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sat, 6 Sep 2025 19:19:08 -0700 Subject: [PATCH] scripts(evals): allow user to talk and only eval when needed --- scripts/evals/eval.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index a27b10aea..fbfef7de5 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -47,7 +47,7 @@ from pipecat.transports.daily.transport import DailyParams, DailyTransport SCRIPT_DIR = Path(__file__).resolve().parent PIPELINE_IDLE_TIMEOUT_SECS = 60 -EVAL_TIMEOUT_SECS = 90 +EVAL_TIMEOUT_SECS = 120 EvalPrompt = str | Tuple[str, ImageFile] @@ -266,8 +266,11 @@ async def run_eval_pipeline( elif isinstance(prompt, tuple): example_prompt, example_image = prompt - eval_prompt = f"The answer is correct if it's appropriate for the context and matches: {eval}." - common_system_prompt = f"Call the eval function with your assessment only if the user answers the question. {eval_prompt}" + eval_prompt = f"The answer is correct if it matches: {eval}." + common_system_prompt = ( + "The user might say things other than the answer and that's allowed. " + f"You should only call the eval function with your assessment when the user actually answers the question. {eval_prompt}" + ) if user_speaks_first: system_prompt = f"You are an LLM eval, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" else: