diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index 6eb8b938c..3a17ed8a6 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -290,9 +290,9 @@ async def run_eval_pipeline( "Ignore greetings, comments, non-answers, or requests for clarification." ) if eval_config.eval_speaks_first: - system_prompt = f"You are an evaluation agent, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" + system_prompt = f"You are an evaluation agent, be extremly brief. Numerical word answers are allowed. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" else: - system_prompt = f"You are an evaluation agent, be extremly brief. First, ask one question: {example_prompt}. {common_system_prompt}" + system_prompt = f"You are an evaluation agent, be extremly brief. Numerical word answers are allowed. First, ask one question: {example_prompt}. {common_system_prompt}" messages = [ { @@ -358,10 +358,6 @@ async def run_eval_pipeline( logger.info(f"Client disconnected") await task.cancel() - @task.event_handler("on_idle_timeout") - async def on_pipeline_idle_timeout(task): - await eval_runner.assert_eval(False) - @task.event_handler("on_pipeline_finished") async def on_pipeline_finished(task, frame): if isinstance(frame, EndFrame): diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index a92e17412..31bc99e5b 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -30,7 +30,7 @@ EVAL_SIMPLE_MATH = EvalConfig( ) EVAL_WEATHER = EvalConfig( - prompt="What's the weather in San Francisco? Temperature should be in any unit, just pick one.", + prompt="What's the weather in San Francisco? Temperature should be in fahrenheits.", eval="The user talks about the weather in San Francisco, including the degrees.", ) diff --git a/src/pipecat/utils/base_object.py b/src/pipecat/utils/base_object.py index bace5e64b..e77f767f2 100644 --- a/src/pipecat/utils/base_object.py +++ b/src/pipecat/utils/base_object.py @@ -13,6 +13,7 @@ and async cleanup for all Pipecat components. import asyncio import inspect +import traceback from abc import ABC from dataclasses import dataclass from typing import Any, Dict, List, Optional @@ -187,7 +188,11 @@ class BaseObject(ABC): else: handler(self, *args, **kwargs) except Exception as e: - logger.error(f"Exception in event handler {event_name}: {e}") + tb = traceback.extract_tb(e.__traceback__) + last = tb[-1] + logger.error( + f"Uncaught exception in event handler '{event_name}' ({last.filename}:{last.lineno}): {e}" + ) def _event_task_finished(self, task: asyncio.Task): """Clean up completed event handler tasks.