From 977a57c8fb6827e3c1349e162f5d8adc3af2d7f0 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 3 Sep 2025 17:11:38 -0400 Subject: [PATCH] Add 14k (CerebrasLLMService) to release evals --- examples/foundational/14k-function-calling-cerebras.py | 10 +++------- scripts/evals/run-release-evals.py | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 642880dd0..8ad80d3ec 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -72,13 +72,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) - # Disabling for now, as we end up in an infinite inference loop with the - # model in this example ("let me check on that" ends up at the end of the - # context, which the model erroneously treats as a nudge to call the tool - # again). - # @llm.event_handler("on_function_calls_started") - # async def on_function_calls_started(service, function_calls): - # await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) weather_function = FunctionSchema( name="get_current_weather", diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 8e4e32419..d2362aebb 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -127,6 +127,7 @@ TESTS_14 = [ ("14h-function-calling-azure.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14i-function-calling-fireworks.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14j-function-calling-nim.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14k-function-calling-cerebras.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14m-function-calling-openrouter.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14n-function-calling-perplexity.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14p-function-calling-gemini-vertex-ai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), @@ -136,7 +137,6 @@ TESTS_14 = [ ("14w-function-calling-mistral.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), # Currently not working. # ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - # ("14k-function-calling-cerebras.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), # ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), # ("14o-function-calling-gemini-openai-format.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ]