scripts(evals): improve eval prompt

examples/foundational/14n-function-calling-perplexity.py

@@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
     messages = [
         {
             "role": "user",
-            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but try to be brief.",
         },
     ]
 

scripts/evals/eval.py

@@ -244,10 +244,10 @@ async def run_eval_pipeline(
 
     llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
 
-    llm.register_function("assert_eval", eval_runner.assert_eval)
+    llm.register_function("eval_function", eval_runner.assert_eval)
 
     eval_function = FunctionSchema(
-        name="assert_eval",
+        name="eval_function",
         description="Called when the user answers a question.",
         properties={
             "result": {
@@ -272,13 +272,15 @@ async def run_eval_pipeline(
         example_prompt, example_image = eval_config.prompt
 
     common_system_prompt = (
-        "The user might say things other than the answer and that's allowed. "
+        "You should only call the eval function if:\n"
-        f"You should only call the eval function when the user: {eval_config.eval}"
+        "- The user explicitly attempts to answer the question, AND\n"
+        f"- Their answer can be cleanly evaluated using: {eval_config.eval}\n"
+        "Ignore greetings, comments, non-answers, or requests for clarification."
     )
     if eval_config.eval_speaks_first:
-        system_prompt = f"You are an LLM eval, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}"
+        system_prompt = f"You are an evaluation agent, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}"
     else:
-        system_prompt = f"You are an LLM eval, be extremly brief. Your goal is to first ask one question: {example_prompt}. {common_system_prompt}"
+        system_prompt = f"You are an evaluation agent, be extremly brief. First, ask one question: {example_prompt}. {common_system_prompt}"
 
     messages = [
         {