Merge pull request #3241 from pipecat-ai/aleix/evals-remove-idle-timeout

evals remove idle timeout and prompt improvements

scripts/evals/eval.py

@@ -290,9 +290,9 @@ async def run_eval_pipeline(
         "Ignore greetings, comments, non-answers, or requests for clarification."
     )
     if eval_config.eval_speaks_first:
-        system_prompt = f"You are an evaluation agent, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}"
+        system_prompt = f"You are an evaluation agent, be extremly brief. Numerical word answers are allowed. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}"
     else:
-        system_prompt = f"You are an evaluation agent, be extremly brief. First, ask one question: {example_prompt}. {common_system_prompt}"
+        system_prompt = f"You are an evaluation agent, be extremly brief. Numerical word answers are allowed. First, ask one question: {example_prompt}. {common_system_prompt}"
 
     messages = [
         {
@@ -358,10 +358,6 @@ async def run_eval_pipeline(
         logger.info(f"Client disconnected")
         await task.cancel()
 
-    @task.event_handler("on_idle_timeout")
-    async def on_pipeline_idle_timeout(task):
-        await eval_runner.assert_eval(False)
-
     @task.event_handler("on_pipeline_finished")
     async def on_pipeline_finished(task, frame):
         if isinstance(frame, EndFrame):

scripts/evals/run-release-evals.py

@@ -30,7 +30,7 @@ EVAL_SIMPLE_MATH = EvalConfig(
 )
 
 EVAL_WEATHER = EvalConfig(
-    prompt="What's the weather in San Francisco? Temperature should be in any unit, just pick one.",
+    prompt="What's the weather in San Francisco? Temperature should be in fahrenheits.",
     eval="The user talks about the weather in San Francisco, including the degrees.",
 )
 

src/pipecat/utils/base_object.py

@@ -13,6 +13,7 @@ and async cleanup for all Pipecat components.
 
 import asyncio
 import inspect
+import traceback
 from abc import ABC
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional
@@ -187,7 +188,11 @@ class BaseObject(ABC):
             else:
                 handler(self, *args, **kwargs)
         except Exception as e:
-            logger.error(f"Exception in event handler {event_name}: {e}")
+            tb = traceback.extract_tb(e.__traceback__)
+            last = tb[-1]
+            logger.error(
+                f"Uncaught exception in event handler '{event_name}' ({last.filename}:{last.lineno}): {e}"
+            )
 
     def _event_task_finished(self, task: asyncio.Task):
         """Clean up completed event handler tasks.