Add 44 to evals, update evals to support user speaking first

examples/foundational/44-voicemail-detection.py

@@ -4,15 +4,13 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-import asyncio
 import os
 
 from dotenv import load_dotenv
 from loguru import logger
 
 from pipecat.audio.vad.silero import SileroVADAnalyzer
-from pipecat.frames.frames import EndFrame, EndTaskFrame, TTSSpeakFrame
-from pipecat.observers.loggers.debug_log_observer import DebugLogObserver
+from pipecat.frames.frames import EndTaskFrame, TTSSpeakFrame
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -63,9 +61,14 @@ async def handle_voicemail(processor):
 
     # Push frames using standard Pipecat pattern
     await processor.push_frame(
-        TTSSpeakFrame("This is Mattie. Call me back when you can!"),
+        TTSSpeakFrame(
+            "Hello, this is Jamie calling about your appointment. Please call me back at 555-0123 when you get this."
+        )
     )
-    await processor.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
+
+    # NOTE: A common pattern is to end pipeline after the voicemail is left.
+    # Uncomment the following line to end the pipeline after leaving the voicemail.
+    # await processor.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
 
 
 async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
@@ -114,22 +117,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
             enable_usage_metrics=True,
         ),
         idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
-        observers=[
-            DebugLogObserver(
-                frame_types={
-                    EndFrame: None,
-                    EndTaskFrame: None,
-                }
-            ),
-        ],
     )
 
     @transport.event_handler("on_client_connected")
     async def on_client_connected(transport, client):
         logger.info(f"Client connected")
-        # # Kick off the conversation.
-        # messages.append({"role": "system", "content": "Please introduce yourself to the user."})
-        # await task.queue_frames([context_aggregator.user().get_context_frame()])
 
     @transport.event_handler("on_client_disconnected")
     async def on_client_disconnected(transport, client):

scripts/evals/eval.py

@@ -89,7 +89,13 @@ class EvalRunner:
     async def assert_eval_false(self):
         await self._queue.put(False)
 
-    async def run_eval(self, example_file: str, prompt: EvalPrompt, eval: Optional[str] = None):
+    async def run_eval(
+        self,
+        example_file: str,
+        prompt: EvalPrompt,
+        eval: Optional[str] = None,
+        user_speaks_first: bool = False,
+    ):
         if not re.match(self._pattern, example_file):
             return
 
@@ -106,7 +112,9 @@ class EvalRunner:
         try:
             tasks = [
                 asyncio.create_task(run_example_pipeline(script_path)),
-                asyncio.create_task(run_eval_pipeline(self, example_file, prompt, eval)),
+                asyncio.create_task(
+                    run_eval_pipeline(self, example_file, prompt, eval, user_speaks_first)
+                ),
             ]
             _, pending = await asyncio.wait(tasks, timeout=EVAL_TIMEOUT_SECS)
             if pending:
@@ -196,6 +204,7 @@ async def run_eval_pipeline(
     example_file: str,
     prompt: EvalPrompt,
     eval: Optional[str],
+    user_speaks_first: bool = False,
 ):
     logger.info(f"Starting eval bot")
 
@@ -225,7 +234,7 @@ async def run_eval_pipeline(
 
     tts = CartesiaTTSService(
         api_key=os.getenv("CARTESIA_API_KEY"),
-        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+        voice_id="97f4b8fb-f2fe-444b-bb9a-c109783a857a",  # Nathan
     )
 
     llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
@@ -260,12 +269,17 @@ async def run_eval_pipeline(
     # See if we need to include an eval prompt.
     eval_prompt = ""
     if eval:
-        eval_prompt = f"The answer is correct if the user says [{eval}]."
+        if user_speaks_first:
+            eval_prompt = f"After the user responds, evaluate if their response is appropriate for the context and matches: [{eval}]."
+            system_prompt = f"You will start the conversation by saying: '{prompt}'. {eval_prompt} Then call the eval function with your assessment."
+        else:
+            eval_prompt = f"The answer is correct if the user says [{eval}]."
+            system_prompt = f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {example_prompt}. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}"
 
     messages = [
         {
             "role": "system",
-            "content": f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {example_prompt}. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}",
+            "content": system_prompt,
         },
     ]
 
@@ -313,6 +327,14 @@ async def run_eval_pipeline(
             )
         await audio_buffer.start_recording()
 
+        # Default behavior is for the bot to speak first
+        # If the eval bot speaks first, we append the prompt to the messages
+        if user_speaks_first:
+            messages.append(
+                {"role": "user", "content": f"Start by saying this exactly: '{prompt}'"}
+            )
+            await task.queue_frames([context_aggregator.user().get_context_frame()])
+
     @transport.event_handler("on_client_disconnected")
     async def on_client_disconnected(transport, client):
         logger.info(f"Client disconnected")

scripts/evals/run-release-evals.py

@@ -24,6 +24,8 @@ ASSETS_DIR = SCRIPT_DIR / "assets"
 
 FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational"
 
+# User speaks first
+USER_SPEAKS_FIRST = True
 
 # Math
 PROMPT_SIMPLE_MATH = "A simple math addition."
@@ -46,6 +48,12 @@ EVAL_SWITCH_LANGUAGE = "Check if the user is now talking in Spanish."
 PROMPT_VISION = ("What do you see?", Image.open(ASSETS_DIR / "cat.jpg"))
 EVAL_VISION = "A cat description."
 
+# Voicemail
+PROMPT_VOICEMAIL = "Please leave a message after the beep."
+EVAL_VOICEMAIL = "Assess the conversation and determine if it is a voicemail."
+PROMPT_CONVERSATION = "Hello, this is Mark."
+EVAL_CONVERSATION = "A start of a conversation, not a voicemail."
+
 TESTS_07 = [
     # 07 series
     ("07-interruptible.py", PROMPT_SIMPLE_MATH, None),
@@ -157,6 +165,11 @@ TESTS_43 = [
     ("43a-heygen-video-service.py", PROMPT_SIMPLE_MATH, None),
 ]
 
+TESTS_44 = [
+    ("44-voicemail-detection.py", PROMPT_VOICEMAIL, EVAL_VOICEMAIL, USER_SPEAKS_FIRST),
+    ("44-voicemail-detection.py", PROMPT_CONVERSATION, EVAL_CONVERSATION, USER_SPEAKS_FIRST),
+]
+
 TESTS = [
     *TESTS_07,
     *TESTS_12,
@@ -168,6 +181,7 @@ TESTS = [
     *TESTS_27,
     *TESTS_40,
     *TESTS_43,
+    *TESTS_44,
 ]
 
 
@@ -189,8 +203,15 @@ async def main(args: argparse.Namespace):
         log_level=log_level,
     )
 
-    for test, prompt, eval in TESTS:
-        await runner.run_eval(test, prompt, eval)
+    # Parse test config: (test, prompt, eval) or (test, prompt, eval, user_speaks_first)
+    for test_config in TESTS:
+        if len(test_config) == 3:
+            test, prompt, eval = test_config
+            user_speaks_first = False
+        else:
+            test, prompt, eval, user_speaks_first = test_config
+
+        await runner.run_eval(test, prompt, eval, user_speaks_first)
 
     runner.print_results()