diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py
index b535d1e7d..3a17ed8a6 100644
--- a/scripts/evals/eval.py
+++ b/scripts/evals/eval.py
@@ -290,9 +290,9 @@ async def run_eval_pipeline(
         "Ignore greetings, comments, non-answers, or requests for clarification."
     )
     if eval_config.eval_speaks_first:
-        system_prompt = f"You are an evaluation agent, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}"
+        system_prompt = f"You are an evaluation agent, be extremly brief. Numerical word answers are allowed. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}"
     else:
-        system_prompt = f"You are an evaluation agent, be extremly brief. First, ask one question: {example_prompt}. {common_system_prompt}"
+        system_prompt = f"You are an evaluation agent, be extremly brief. Numerical word answers are allowed. First, ask one question: {example_prompt}. {common_system_prompt}"
 
     messages = [
         {
diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py
index a92e17412..31bc99e5b 100644
--- a/scripts/evals/run-release-evals.py
+++ b/scripts/evals/run-release-evals.py
@@ -30,7 +30,7 @@ EVAL_SIMPLE_MATH = EvalConfig(
 )
 
 EVAL_WEATHER = EvalConfig(
-    prompt="What's the weather in San Francisco? Temperature should be in any unit, just pick one.",
+    prompt="What's the weather in San Francisco? Temperature should be in fahrenheits.",
     eval="The user talks about the weather in San Francisco, including the degrees.",
 )