From 5b30f1b1ef6a0b16fa291689b7f75046508d452b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 16 Dec 2025 17:26:50 -0800 Subject: [PATCH] scripts(evals): improve prompts --- scripts/evals/eval.py | 4 ++-- scripts/evals/run-release-evals.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index b535d1e7d..3a17ed8a6 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -290,9 +290,9 @@ async def run_eval_pipeline( "Ignore greetings, comments, non-answers, or requests for clarification." ) if eval_config.eval_speaks_first: - system_prompt = f"You are an evaluation agent, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" + system_prompt = f"You are an evaluation agent, be extremly brief. Numerical word answers are allowed. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" else: - system_prompt = f"You are an evaluation agent, be extremly brief. First, ask one question: {example_prompt}. {common_system_prompt}" + system_prompt = f"You are an evaluation agent, be extremly brief. Numerical word answers are allowed. First, ask one question: {example_prompt}. {common_system_prompt}" messages = [ { diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index a92e17412..31bc99e5b 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -30,7 +30,7 @@ EVAL_SIMPLE_MATH = EvalConfig( ) EVAL_WEATHER = EvalConfig( - prompt="What's the weather in San Francisco? Temperature should be in any unit, just pick one.", + prompt="What's the weather in San Francisco? Temperature should be in fahrenheits.", eval="The user talks about the weather in San Francisco, including the degrees.", )