From 4f1468e0fabc21e7508aa4492c45c1af3013daff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 7 Nov 2025 10:05:27 -0800 Subject: [PATCH] scripts(evals): improve eval prompt --- .../14n-function-calling-perplexity.py | 2 +- scripts/evals/eval.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index 2f37768b7..32ce47150 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "user", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but try to be brief.", }, ] diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index e23a99420..306385988 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -244,10 +244,10 @@ async def run_eval_pipeline( llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - llm.register_function("assert_eval", eval_runner.assert_eval) + llm.register_function("eval_function", eval_runner.assert_eval) eval_function = FunctionSchema( - name="assert_eval", + name="eval_function", description="Called when the user answers a question.", properties={ "result": { @@ -272,13 +272,15 @@ async def run_eval_pipeline( example_prompt, example_image = eval_config.prompt common_system_prompt = ( - "The user might say things other than the answer and that's allowed. " - f"You should only call the eval function when the user: {eval_config.eval}" + "You should only call the eval function if:\n" + "- The user explicitly attempts to answer the question, AND\n" + f"- Their answer can be cleanly evaluated using: {eval_config.eval}\n" + "Ignore greetings, comments, non-answers, or requests for clarification." ) if eval_config.eval_speaks_first: - system_prompt = f"You are an LLM eval, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" + system_prompt = f"You are an evaluation agent, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" else: - system_prompt = f"You are an LLM eval, be extremly brief. Your goal is to first ask one question: {example_prompt}. {common_system_prompt}" + system_prompt = f"You are an evaluation agent, be extremly brief. First, ask one question: {example_prompt}. {common_system_prompt}" messages = [ {