wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add evals for parallel function calling

Browse Source
This commit is contained in:
Paul Kompfner
2026-01-12 17:00:13 -05:00
parent 8812686b17
commit 6668712f7b
1 changed files with 13 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
scripts/evals/run-release-evals.py
View File

@@ -30,10 +30,15 @@ EVAL_SIMPLE_MATH = EvalConfig(
) )
EVAL_WEATHER = EvalConfig( EVAL_WEATHER = EvalConfig(
prompt="What's the weather in San Francisco? Temperature should be in fahrenheits.", prompt="What's the weather in San Francisco? Temperature should be in Fahrenheit.",
eval="The user talks about the weather in San Francisco, including the degrees.", eval="The user talks about the weather in San Francisco, including the degrees.",
) )
EVAL_WEATHER_AND_RESTAURANT = EvalConfig(
prompt="What's the weather in San Francisco, and what's a good restaurant there? Temperature should be in Fahrenheit.",
eval="The user talks about the weather in San Francisco, including the degrees, and provides a restaurant recommendation.",
)
EVAL_ONLINE_SEARCH = EvalConfig( EVAL_ONLINE_SEARCH = EvalConfig(
prompt="What's the current date in UTC?", prompt="What's the current date in UTC?",
eval=f"Current date in UTC is {datetime.now(timezone.utc).strftime('%A, %B %d, %Y')}.", eval=f"Current date in UTC is {datetime.now(timezone.utc).strftime('%A, %B %d, %Y')}.",
@@ -145,10 +150,16 @@ TESTS_12 = [
("12d-describe-image-moondream.py", EVAL_VISION_IMAGE()), ("12d-describe-image-moondream.py", EVAL_VISION_IMAGE()),
] ]
# For a few major services, we also test parallel function calling.
# (We don't bother doing this with every single service, as it's expensive and
# most rely on the same OpenAI-compatible implementation.)
TESTS_14 = [ TESTS_14 = [
("14-function-calling.py", EVAL_WEATHER), ("14-function-calling.py", EVAL_WEATHER),
("14-function-calling.py", EVAL_WEATHER_AND_RESTAURANT),
("14a-function-calling-anthropic.py", EVAL_WEATHER), ("14a-function-calling-anthropic.py", EVAL_WEATHER),
("14a-function-calling-anthropic.py", EVAL_WEATHER_AND_RESTAURANT),
("14e-function-calling-google.py", EVAL_WEATHER), ("14e-function-calling-google.py", EVAL_WEATHER),
("14e-function-calling-google.py", EVAL_WEATHER_AND_RESTAURANT),
("14f-function-calling-groq.py", EVAL_WEATHER), ("14f-function-calling-groq.py", EVAL_WEATHER),
("14g-function-calling-grok.py", EVAL_WEATHER), ("14g-function-calling-grok.py", EVAL_WEATHER),
("14h-function-calling-azure.py", EVAL_WEATHER), ("14h-function-calling-azure.py", EVAL_WEATHER),
@@ -160,6 +171,7 @@ TESTS_14 = [
("14p-function-calling-gemini-vertex-ai.py", EVAL_WEATHER), ("14p-function-calling-gemini-vertex-ai.py", EVAL_WEATHER),
("14q-function-calling-qwen.py", EVAL_WEATHER), ("14q-function-calling-qwen.py", EVAL_WEATHER),
("14r-function-calling-aws.py", EVAL_WEATHER), ("14r-function-calling-aws.py", EVAL_WEATHER),
("14r-function-calling-aws.py", EVAL_WEATHER_AND_RESTAURANT),
("14v-function-calling-openai.py", EVAL_WEATHER), ("14v-function-calling-openai.py", EVAL_WEATHER),
("14w-function-calling-mistral.py", EVAL_WEATHER), ("14w-function-calling-mistral.py", EVAL_WEATHER),
("14x-function-calling-openpipe.py", EVAL_WEATHER), ("14x-function-calling-openpipe.py", EVAL_WEATHER),