From 1c1ee940749f974ad83734fa0c4853257043609a Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 9 Aug 2025 13:38:05 -0400 Subject: [PATCH] Add 44 to evals, update evals to support user speaking first --- .../foundational/44-voicemail-detection.py | 24 +++++--------- scripts/evals/eval.py | 32 ++++++++++++++++--- scripts/evals/run-release-evals.py | 25 +++++++++++++-- 3 files changed, 58 insertions(+), 23 deletions(-) diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index 358116aed..f2957309d 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -4,15 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import EndFrame, EndTaskFrame, TTSSpeakFrame -from pipecat.observers.loggers.debug_log_observer import DebugLogObserver +from pipecat.frames.frames import EndTaskFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -63,9 +61,14 @@ async def handle_voicemail(processor): # Push frames using standard Pipecat pattern await processor.push_frame( - TTSSpeakFrame("This is Mattie. Call me back when you can!"), + TTSSpeakFrame( + "Hello, this is Jamie calling about your appointment. Please call me back at 555-0123 when you get this." + ) ) - await processor.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) + + # NOTE: A common pattern is to end pipeline after the voicemail is left. + # Uncomment the following line to end the pipeline after leaving the voicemail. + # await processor.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @@ -114,22 +117,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): enable_usage_metrics=True, ), idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - observers=[ - DebugLogObserver( - frame_types={ - EndFrame: None, - EndTaskFrame: None, - } - ), - ], ) @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") - # # Kick off the conversation. - # messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - # await task.queue_frames([context_aggregator.user().get_context_frame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index b91f27c6e..bbc170803 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -89,7 +89,13 @@ class EvalRunner: async def assert_eval_false(self): await self._queue.put(False) - async def run_eval(self, example_file: str, prompt: EvalPrompt, eval: Optional[str] = None): + async def run_eval( + self, + example_file: str, + prompt: EvalPrompt, + eval: Optional[str] = None, + user_speaks_first: bool = False, + ): if not re.match(self._pattern, example_file): return @@ -106,7 +112,9 @@ class EvalRunner: try: tasks = [ asyncio.create_task(run_example_pipeline(script_path)), - asyncio.create_task(run_eval_pipeline(self, example_file, prompt, eval)), + asyncio.create_task( + run_eval_pipeline(self, example_file, prompt, eval, user_speaks_first) + ), ] _, pending = await asyncio.wait(tasks, timeout=EVAL_TIMEOUT_SECS) if pending: @@ -196,6 +204,7 @@ async def run_eval_pipeline( example_file: str, prompt: EvalPrompt, eval: Optional[str], + user_speaks_first: bool = False, ): logger.info(f"Starting eval bot") @@ -225,7 +234,7 @@ async def run_eval_pipeline( tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + voice_id="97f4b8fb-f2fe-444b-bb9a-c109783a857a", # Nathan ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) @@ -260,12 +269,17 @@ async def run_eval_pipeline( # See if we need to include an eval prompt. eval_prompt = "" if eval: - eval_prompt = f"The answer is correct if the user says [{eval}]." + if user_speaks_first: + eval_prompt = f"After the user responds, evaluate if their response is appropriate for the context and matches: [{eval}]." + system_prompt = f"You will start the conversation by saying: '{prompt}'. {eval_prompt} Then call the eval function with your assessment." + else: + eval_prompt = f"The answer is correct if the user says [{eval}]." + system_prompt = f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {example_prompt}. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}" messages = [ { "role": "system", - "content": f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {example_prompt}. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}", + "content": system_prompt, }, ] @@ -313,6 +327,14 @@ async def run_eval_pipeline( ) await audio_buffer.start_recording() + # Default behavior is for the bot to speak first + # If the eval bot speaks first, we append the prompt to the messages + if user_speaks_first: + messages.append( + {"role": "user", "content": f"Start by saying this exactly: '{prompt}'"} + ) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 49acc432a..f30b71193 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -24,6 +24,8 @@ ASSETS_DIR = SCRIPT_DIR / "assets" FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational" +# User speaks first +USER_SPEAKS_FIRST = True # Math PROMPT_SIMPLE_MATH = "A simple math addition." @@ -46,6 +48,12 @@ EVAL_SWITCH_LANGUAGE = "Check if the user is now talking in Spanish." PROMPT_VISION = ("What do you see?", Image.open(ASSETS_DIR / "cat.jpg")) EVAL_VISION = "A cat description." +# Voicemail +PROMPT_VOICEMAIL = "Please leave a message after the beep." +EVAL_VOICEMAIL = "Assess the conversation and determine if it is a voicemail." +PROMPT_CONVERSATION = "Hello, this is Mark." +EVAL_CONVERSATION = "A start of a conversation, not a voicemail." + TESTS_07 = [ # 07 series ("07-interruptible.py", PROMPT_SIMPLE_MATH, None), @@ -157,6 +165,11 @@ TESTS_43 = [ ("43a-heygen-video-service.py", PROMPT_SIMPLE_MATH, None), ] +TESTS_44 = [ + ("44-voicemail-detection.py", PROMPT_VOICEMAIL, EVAL_VOICEMAIL, USER_SPEAKS_FIRST), + ("44-voicemail-detection.py", PROMPT_CONVERSATION, EVAL_CONVERSATION, USER_SPEAKS_FIRST), +] + TESTS = [ *TESTS_07, *TESTS_12, @@ -168,6 +181,7 @@ TESTS = [ *TESTS_27, *TESTS_40, *TESTS_43, + *TESTS_44, ] @@ -189,8 +203,15 @@ async def main(args: argparse.Namespace): log_level=log_level, ) - for test, prompt, eval in TESTS: - await runner.run_eval(test, prompt, eval) + # Parse test config: (test, prompt, eval) or (test, prompt, eval, user_speaks_first) + for test_config in TESTS: + if len(test_config) == 3: + test, prompt, eval = test_config + user_speaks_first = False + else: + test, prompt, eval, user_speaks_first = test_config + + await runner.run_eval(test, prompt, eval, user_speaks_first) runner.print_results()