From a5e42337a4c2e479746471edaeeffd21549a236c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 14 Dec 2025 12:12:34 -0800 Subject: [PATCH 1/4] frames: EndFrame and CancelFrame reason is now Any --- changelog/3231.changed.md | 3 +++ src/pipecat/frames/frames.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 changelog/3231.changed.md diff --git a/changelog/3231.changed.md b/changelog/3231.changed.md new file mode 100644 index 000000000..02de3cf8f --- /dev/null +++ b/changelog/3231.changed.md @@ -0,0 +1,3 @@ +- Changed the `reason` field in `EndFrame`, `CancelFrame`, `EndTaskFrame`, and + `CancelTaskFrame` from `str` to `Any` to indicate that it can hold values + other than strings. diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 542f21fc0..c7d07dbed 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -947,7 +947,7 @@ class CancelFrame(SystemFrame): reason: Optional reason for pushing a cancel frame. """ - reason: Optional[str] = None + reason: Optional[Any] = None def __str__(self): return f"{self.name}(reason: {self.reason})" @@ -1537,7 +1537,7 @@ class EndTaskFrame(TaskFrame): reason: Optional reason for pushing an end frame. """ - reason: Optional[str] = None + reason: Optional[Any] = None def __str__(self): return f"{self.name}(reason: {self.reason})" @@ -1555,7 +1555,7 @@ class CancelTaskFrame(TaskFrame): reason: Optional reason for pushing a cancel frame. """ - reason: Optional[str] = None + reason: Optional[Any] = None def __str__(self): return f"{self.name}(reason: {self.reason})" @@ -1634,7 +1634,7 @@ class EndFrame(ControlFrame): reason: Optional reason for pushing an end frame. """ - reason: Optional[str] = None + reason: Optional[Any] = None def __str__(self): return f"{self.name}(reason: {self.reason})" From a14c911fb2d3db7f9c3b5ec1f327915a31f868f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 14 Dec 2025 12:15:45 -0800 Subject: [PATCH 2/4] scripts(evals): improve eval assertion on exit --- .../26a-gemini-live-transcription.py | 1 - .../26b-gemini-live-function-calling.py | 1 - .../foundational/26c-gemini-live-video.py | 1 - scripts/evals/eval.py | 32 +++++++++++++------ 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/examples/foundational/26a-gemini-live-transcription.py b/examples/foundational/26a-gemini-live-transcription.py index 2aed3e650..7d0f87ded 100644 --- a/examples/foundational/26a-gemini-live-transcription.py +++ b/examples/foundational/26a-gemini-live-transcription.py @@ -17,7 +17,6 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments diff --git a/examples/foundational/26b-gemini-live-function-calling.py b/examples/foundational/26b-gemini-live-function-calling.py index 41c312822..fe7afbc6d 100644 --- a/examples/foundational/26b-gemini-live-function-calling.py +++ b/examples/foundational/26b-gemini-live-function-calling.py @@ -20,7 +20,6 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport diff --git a/examples/foundational/26c-gemini-live-video.py b/examples/foundational/26c-gemini-live-video.py index 3b68a650a..d42aaa385 100644 --- a/examples/foundational/26c-gemini-live-video.py +++ b/examples/foundational/26c-gemini-live-video.py @@ -18,7 +18,6 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index 306385988..db70820a0 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -31,7 +31,13 @@ from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import EndTaskFrame, LLMRunFrame, OutputImageRawFrame +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + EndTaskFrame, + LLMRunFrame, + OutputImageRawFrame, +) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -50,6 +56,7 @@ SCRIPT_DIR = Path(__file__).resolve().parent PIPELINE_IDLE_TIMEOUT_SECS = 60 EVAL_TIMEOUT_SECS = 120 +EVAL_RESULT_TIMEOUT_SECS = 10 EvalPrompt = str | Tuple[str, ImageFile] @@ -88,16 +95,15 @@ class EvalRunner: os.makedirs(self._logs_dir, exist_ok=True) os.makedirs(self._recordings_dir, exist_ok=True) - async def assert_eval(self, params: FunctionCallParams): + async def function_assert_eval(self, params: FunctionCallParams): result = params.arguments["result"] reasoning = params.arguments["reasoning"] logger.debug(f"🧠 EVAL REASONING(result: {result}): {reasoning}") - await self._queue.put(result) await params.result_callback(None) - await params.llm.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) + await params.llm.push_frame(EndTaskFrame(reason=result), FrameDirection.UPSTREAM) - async def assert_eval_false(self): - await self._queue.put(False) + async def assert_eval(self, result: bool): + await self._queue.put(result) async def run_eval( self, @@ -136,8 +142,9 @@ class EvalRunner: logger.error(f"ERROR: Unable to run {example_file}: {e}") try: - result = await asyncio.wait_for(self._queue.get(), timeout=1.0) + result = await asyncio.wait_for(self._queue.get(), timeout=EVAL_RESULT_TIMEOUT_SECS) except asyncio.TimeoutError: + logger.error(f"ERROR: Timeout waiting for eval result.") result = False if result: @@ -244,7 +251,7 @@ async def run_eval_pipeline( llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - llm.register_function("eval_function", eval_runner.assert_eval) + llm.register_function("eval_function", eval_runner.function_assert_eval) eval_function = FunctionSchema( name="eval_function", @@ -348,7 +355,14 @@ async def run_eval_pipeline( @task.event_handler("on_idle_timeout") async def on_pipeline_idle_timeout(task): - await eval_runner.assert_eval_false() + await eval_runner.assert_eval(False) + + @task.event_handler("on_pipeline_finished") + async def on_pipeline_finished(task, frame): + if isinstance(frame, EndFrame): + await eval_runner.assert_eval(frame.reason) + elif isinstance(frame, CancelFrame): + await eval_runner.assert_eval(False) # TODO(aleix): We should handle SIGINT and SIGTERM so we can cancel both the # eval and the example. From 70a80847a7472679442c0dd52fcececa98aa1193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 14 Dec 2025 16:26:28 -0800 Subject: [PATCH 3/4] scripts(evals): use future instead of a queue to store eval result --- scripts/evals/eval.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index db70820a0..6eb8b938c 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -85,7 +85,7 @@ class EvalRunner: self._log_level = log_level self._total_success = 0 self._tests: List[EvalResult] = [] - self._queue = asyncio.Queue() + self._result_future: Optional[asyncio.Future[bool]] = None # We to save runner files. name = name or f"{datetime.now().strftime('%Y%m%d_%H%M%S')}" @@ -103,7 +103,8 @@ class EvalRunner: await params.llm.push_frame(EndTaskFrame(reason=result), FrameDirection.UPSTREAM) async def assert_eval(self, result: bool): - await self._queue.put(result) + if self._result_future: + self._result_future.set_result(result) async def run_eval( self, @@ -123,6 +124,9 @@ class EvalRunner: start_time = time.time() + # Create a future to store the eval result. + self._result_future = asyncio.get_running_loop().create_future() + try: tasks = [ asyncio.create_task(run_example_pipeline(script_path, eval_config)), @@ -142,7 +146,8 @@ class EvalRunner: logger.error(f"ERROR: Unable to run {example_file}: {e}") try: - result = await asyncio.wait_for(self._queue.get(), timeout=EVAL_RESULT_TIMEOUT_SECS) + # Wait for the future to resolve. + result = await asyncio.wait_for(self._result_future, timeout=EVAL_RESULT_TIMEOUT_SECS) except asyncio.TimeoutError: logger.error(f"ERROR: Timeout waiting for eval result.") result = False From 21e346abe274c5902bb83d5ce90337668c77981b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 14 Dec 2025 18:22:25 -0800 Subject: [PATCH 4/4] scripts(evals): improve eval prompts --- scripts/evals/run-release-evals.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 9e0d88260..fb75a0c03 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -30,13 +30,13 @@ EVAL_SIMPLE_MATH = EvalConfig( ) EVAL_WEATHER = EvalConfig( - prompt="What's the weather in San Francisco (in farhenheit or celsius)?", - eval="The user says something specific about the current weather in San Francisco, including the degrees (in farhenheit or celsius).", + prompt="What's the weather in San Francisco? Temperature should be in any unit, just pick one.", + eval="The user talks about the weather in San Francisco, including the degrees.", ) EVAL_ONLINE_SEARCH = EvalConfig( - prompt="What's the date right now in London?", - eval=f"The user says today is {datetime.now(timezone.utc).strftime('%B %d, %Y')} in London.", + prompt="What's the current date in UTC?", + eval=f"Current date in UTC is {datetime.now(timezone.utc).strftime('%A, %B %d, %Y')}.", ) EVAL_SWITCH_LANGUAGE = EvalConfig(