Add 44 to evals, update evals to support user speaking first

scripts/evals/eval.py

@@ -89,7 +89,13 @@ class EvalRunner:
     async def assert_eval_false(self):
         await self._queue.put(False)
 
-    async def run_eval(self, example_file: str, prompt: EvalPrompt, eval: Optional[str] = None):
+    async def run_eval(
+        self,
+        example_file: str,
+        prompt: EvalPrompt,
+        eval: Optional[str] = None,
+        user_speaks_first: bool = False,
+    ):
         if not re.match(self._pattern, example_file):
             return
 
@@ -106,7 +112,9 @@ class EvalRunner:
         try:
             tasks = [
                 asyncio.create_task(run_example_pipeline(script_path)),
-                asyncio.create_task(run_eval_pipeline(self, example_file, prompt, eval)),
+                asyncio.create_task(
+                    run_eval_pipeline(self, example_file, prompt, eval, user_speaks_first)
+                ),
             ]
             _, pending = await asyncio.wait(tasks, timeout=EVAL_TIMEOUT_SECS)
             if pending:
@@ -196,6 +204,7 @@ async def run_eval_pipeline(
     example_file: str,
     prompt: EvalPrompt,
     eval: Optional[str],
+    user_speaks_first: bool = False,
 ):
     logger.info(f"Starting eval bot")
 
@@ -225,7 +234,7 @@ async def run_eval_pipeline(
 
     tts = CartesiaTTSService(
         api_key=os.getenv("CARTESIA_API_KEY"),
-        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+        voice_id="97f4b8fb-f2fe-444b-bb9a-c109783a857a",  # Nathan
     )
 
     llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
@@ -260,12 +269,17 @@ async def run_eval_pipeline(
     # See if we need to include an eval prompt.
     eval_prompt = ""
     if eval:
-        eval_prompt = f"The answer is correct if the user says [{eval}]."
+        if user_speaks_first:
+            eval_prompt = f"After the user responds, evaluate if their response is appropriate for the context and matches: [{eval}]."
+            system_prompt = f"You will start the conversation by saying: '{prompt}'. {eval_prompt} Then call the eval function with your assessment."
+        else:
+            eval_prompt = f"The answer is correct if the user says [{eval}]."
+            system_prompt = f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {example_prompt}. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}"
 
     messages = [
         {
             "role": "system",
-            "content": f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {example_prompt}. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}",
+            "content": system_prompt,
         },
     ]
 
@@ -313,6 +327,14 @@ async def run_eval_pipeline(
             )
         await audio_buffer.start_recording()
 
+        # Default behavior is for the bot to speak first
+        # If the eval bot speaks first, we append the prompt to the messages
+        if user_speaks_first:
+            messages.append(
+                {"role": "user", "content": f"Start by saying this exactly: '{prompt}'"}
+            )
+            await task.queue_frames([context_aggregator.user().get_context_frame()])
+
     @transport.event_handler("on_client_disconnected")
     async def on_client_disconnected(transport, client):
         logger.info(f"Client disconnected")