From 080da8b94cb9ce5c96d444240068274ed14893ad Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 31 Mar 2026 22:09:42 -0400 Subject: [PATCH] Update eval script paths to match renamed example files --- scripts/evals/run-release-evals.py | 256 ++++++++++++++--------------- 1 file changed, 128 insertions(+), 128 deletions(-) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index be3bd1295..3a6055772 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -97,70 +97,70 @@ EVAL_COMPLETE_TURN = EvalConfig( TESTS_VOICE = [ - ("voice/cartesia.py", EVAL_SIMPLE_MATH), - ("voice/cartesia-http.py", EVAL_SIMPLE_MATH), - ("voice/speechmatics.py", EVAL_SIMPLE_MATH), - ("voice/speechmatics-vad.py", EVAL_SIMPLE_MATH), - ("voice/langchain.py", EVAL_SIMPLE_MATH), - ("voice/deepgram.py", EVAL_SIMPLE_MATH), - ("voice/deepgram-flux.py", EVAL_SIMPLE_MATH), - ("voice/deepgram-http.py", EVAL_SIMPLE_MATH), - ("voice/elevenlabs.py", EVAL_SIMPLE_MATH), - ("voice/elevenlabs-http.py", EVAL_SIMPLE_MATH), - ("voice/xai.py", EVAL_SIMPLE_MATH), - ("voice/azure.py", EVAL_SIMPLE_MATH), - ("voice/azure-http.py", EVAL_SIMPLE_MATH), - ("voice/openai.py", EVAL_SIMPLE_MATH), - ("voice/openai-http.py", EVAL_SIMPLE_MATH), - ("voice/gladia.py", EVAL_SIMPLE_MATH), - ("voice/gladia-vad.py", EVAL_SIMPLE_MATH), - ("voice/lmnt.py", EVAL_SIMPLE_MATH), - ("voice/groq.py", EVAL_SIMPLE_MATH), - ("voice/aws.py", EVAL_SIMPLE_MATH), - ("voice/aws-strands.py", EVAL_WEATHER), - ("voice/google-gemini-tts.py", EVAL_SIMPLE_MATH), - ("voice/google.py", EVAL_SIMPLE_MATH), - ("voice/google-http.py", EVAL_SIMPLE_MATH), - ("voice/assemblyai.py", EVAL_SIMPLE_MATH), - ("voice/krisp-viva.py", EVAL_SIMPLE_MATH), - ("voice/rime.py", EVAL_SIMPLE_MATH), - ("voice/rime-http.py", EVAL_SIMPLE_MATH), - ("voice/nvidia.py", EVAL_SIMPLE_MATH), - ("voice/google-audio-in.py", EVAL_SIMPLE_MATH), - ("voice/fish.py", EVAL_SIMPLE_MATH), - ("voice/neuphonic.py", EVAL_SIMPLE_MATH), - ("voice/neuphonic-http.py", EVAL_SIMPLE_MATH), - ("voice/fal.py", EVAL_SIMPLE_MATH), - ("voice/minimax.py", EVAL_SIMPLE_MATH), - ("voice/sarvam.py", EVAL_SIMPLE_MATH), - ("voice/sarvam-http.py", EVAL_SIMPLE_MATH), - ("voice/soniox.py", EVAL_SIMPLE_MATH), - ("voice/inworld.py", EVAL_SIMPLE_MATH), - ("voice/inworld-http.py", EVAL_SIMPLE_MATH), - ("voice/asyncai.py", EVAL_SIMPLE_MATH), - ("voice/asyncai-http.py", EVAL_SIMPLE_MATH), - ("voice/aicoustics.py", EVAL_SIMPLE_MATH), - ("voice/hume.py", EVAL_SIMPLE_MATH), - ("voice/gradium.py", EVAL_SIMPLE_MATH), - ("voice/camb.py", EVAL_SIMPLE_MATH), - ("voice/piper.py", EVAL_SIMPLE_MATH), - ("voice/kokoro.py", EVAL_SIMPLE_MATH), - ("voice/resemble.py", EVAL_SIMPLE_MATH), - ("voice/smallest.py", EVAL_SIMPLE_MATH), - ("voice/openai-responses.py", EVAL_SIMPLE_MATH), - ("voice/openai-responses-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-cartesia.py", EVAL_SIMPLE_MATH), + ("voice/voice-cartesia-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-speechmatics.py", EVAL_SIMPLE_MATH), + ("voice/voice-speechmatics-vad.py", EVAL_SIMPLE_MATH), + ("voice/voice-langchain.py", EVAL_SIMPLE_MATH), + ("voice/voice-deepgram.py", EVAL_SIMPLE_MATH), + ("voice/voice-deepgram-flux.py", EVAL_SIMPLE_MATH), + ("voice/voice-deepgram-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-elevenlabs.py", EVAL_SIMPLE_MATH), + ("voice/voice-elevenlabs-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-xai.py", EVAL_SIMPLE_MATH), + ("voice/voice-azure.py", EVAL_SIMPLE_MATH), + ("voice/voice-azure-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-openai.py", EVAL_SIMPLE_MATH), + ("voice/voice-openai-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-gladia.py", EVAL_SIMPLE_MATH), + ("voice/voice-gladia-vad.py", EVAL_SIMPLE_MATH), + ("voice/voice-lmnt.py", EVAL_SIMPLE_MATH), + ("voice/voice-groq.py", EVAL_SIMPLE_MATH), + ("voice/voice-aws.py", EVAL_SIMPLE_MATH), + ("voice/voice-aws-strands.py", EVAL_WEATHER), + ("voice/voice-google-gemini-tts.py", EVAL_SIMPLE_MATH), + ("voice/voice-google.py", EVAL_SIMPLE_MATH), + ("voice/voice-google-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-assemblyai.py", EVAL_SIMPLE_MATH), + ("voice/voice-krisp-viva.py", EVAL_SIMPLE_MATH), + ("voice/voice-rime.py", EVAL_SIMPLE_MATH), + ("voice/voice-rime-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-nvidia.py", EVAL_SIMPLE_MATH), + ("voice/voice-google-audio-in.py", EVAL_SIMPLE_MATH), + ("voice/voice-fish.py", EVAL_SIMPLE_MATH), + ("voice/voice-neuphonic.py", EVAL_SIMPLE_MATH), + ("voice/voice-neuphonic-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-fal.py", EVAL_SIMPLE_MATH), + ("voice/voice-minimax.py", EVAL_SIMPLE_MATH), + ("voice/voice-sarvam.py", EVAL_SIMPLE_MATH), + ("voice/voice-sarvam-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-soniox.py", EVAL_SIMPLE_MATH), + ("voice/voice-inworld.py", EVAL_SIMPLE_MATH), + ("voice/voice-inworld-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-asyncai.py", EVAL_SIMPLE_MATH), + ("voice/voice-asyncai-http.py", EVAL_SIMPLE_MATH), + ("voice/voice-aicoustics.py", EVAL_SIMPLE_MATH), + ("voice/voice-hume.py", EVAL_SIMPLE_MATH), + ("voice/voice-gradium.py", EVAL_SIMPLE_MATH), + ("voice/voice-camb.py", EVAL_SIMPLE_MATH), + ("voice/voice-piper.py", EVAL_SIMPLE_MATH), + ("voice/voice-kokoro.py", EVAL_SIMPLE_MATH), + ("voice/voice-resemble.py", EVAL_SIMPLE_MATH), + ("voice/voice-smallest.py", EVAL_SIMPLE_MATH), + ("voice/voice-openai-responses.py", EVAL_SIMPLE_MATH), + ("voice/voice-openai-responses-http.py", EVAL_SIMPLE_MATH), # Needs a local XTTS docker instance running. - # ("voice/xtts.py", EVAL_SIMPLE_MATH), + # ("voice/voice-xtts.py", EVAL_SIMPLE_MATH), ] TESTS_VISION = [ - ("vision/openai.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), - ("vision/openai-responses.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), - ("vision/openai-responses-http.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), - ("vision/anthropic.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), - ("vision/aws.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), - ("vision/gemini-flash.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), - ("vision/moondream.py", EVAL_VISION_IMAGE()), + ("vision/vision-openai.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("vision/vision-openai-responses.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("vision/vision-openai-responses-http.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("vision/vision-anthropic.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("vision/vision-aws.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("vision/vision-gemini-flash.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("vision/vision-moondream.py", EVAL_VISION_IMAGE()), ] # For a few major services, we also test parallel function calling. @@ -169,91 +169,91 @@ TESTS_VISION = [ TESTS_FUNCTION_CALLING = [ ("getting-started/07-function-calling.py", EVAL_WEATHER), ("getting-started/07-function-calling.py", EVAL_WEATHER_AND_RESTAURANT), - ("function-calling/openai-responses.py", EVAL_WEATHER), - ("function-calling/openai-responses.py", EVAL_WEATHER_AND_RESTAURANT), - ("function-calling/openai-responses-http.py", EVAL_WEATHER), - ("function-calling/openai-responses-http.py", EVAL_WEATHER_AND_RESTAURANT), - ("function-calling/anthropic.py", EVAL_WEATHER), - ("function-calling/anthropic.py", EVAL_WEATHER_AND_RESTAURANT), - ("function-calling/openai.py", EVAL_WEATHER), - ("function-calling/google.py", EVAL_WEATHER), - ("function-calling/google.py", EVAL_WEATHER_AND_RESTAURANT), - ("function-calling/groq.py", EVAL_WEATHER), - ("function-calling/grok.py", EVAL_WEATHER), - ("function-calling/azure.py", EVAL_WEATHER), - ("function-calling/fireworks.py", EVAL_WEATHER), - ("function-calling/nvidia.py", EVAL_WEATHER), - ("function-calling/cerebras.py", EVAL_WEATHER), - ("function-calling/openrouter.py", EVAL_WEATHER), - ("function-calling/perplexity.py", EVAL_WEATHER), - ("function-calling/google-vertex.py", EVAL_WEATHER), - ("function-calling/qwen.py", EVAL_WEATHER), - ("function-calling/aws.py", EVAL_WEATHER), - ("function-calling/sambanova.py", EVAL_WEATHER), - ("function-calling/aws.py", EVAL_WEATHER_AND_RESTAURANT), - ("function-calling/nebius.py", EVAL_WEATHER), - ("function-calling/mistral.py", EVAL_WEATHER), - ("function-calling/sarvam.py", EVAL_WEATHER), - ("function-calling/novita.py", EVAL_WEATHER), + ("function-calling/function-calling-openai-responses.py", EVAL_WEATHER), + ("function-calling/function-calling-openai-responses.py", EVAL_WEATHER_AND_RESTAURANT), + ("function-calling/function-calling-openai-responses-http.py", EVAL_WEATHER), + ("function-calling/function-calling-openai-responses-http.py", EVAL_WEATHER_AND_RESTAURANT), + ("function-calling/function-calling-anthropic.py", EVAL_WEATHER), + ("function-calling/function-calling-anthropic.py", EVAL_WEATHER_AND_RESTAURANT), + ("function-calling/function-calling-openai.py", EVAL_WEATHER), + ("function-calling/function-calling-google.py", EVAL_WEATHER), + ("function-calling/function-calling-google.py", EVAL_WEATHER_AND_RESTAURANT), + ("function-calling/function-calling-groq.py", EVAL_WEATHER), + ("function-calling/function-calling-grok.py", EVAL_WEATHER), + ("function-calling/function-calling-azure.py", EVAL_WEATHER), + ("function-calling/function-calling-fireworks.py", EVAL_WEATHER), + ("function-calling/function-calling-nvidia.py", EVAL_WEATHER), + ("function-calling/function-calling-cerebras.py", EVAL_WEATHER), + ("function-calling/function-calling-openrouter.py", EVAL_WEATHER), + ("function-calling/function-calling-perplexity.py", EVAL_WEATHER), + ("function-calling/function-calling-google-vertex.py", EVAL_WEATHER), + ("function-calling/function-calling-qwen.py", EVAL_WEATHER), + ("function-calling/function-calling-aws.py", EVAL_WEATHER), + ("function-calling/function-calling-sambanova.py", EVAL_WEATHER), + ("function-calling/function-calling-aws.py", EVAL_WEATHER_AND_RESTAURANT), + ("function-calling/function-calling-nebius.py", EVAL_WEATHER), + ("function-calling/function-calling-mistral.py", EVAL_WEATHER), + ("function-calling/function-calling-sarvam.py", EVAL_WEATHER), + ("function-calling/function-calling-novita.py", EVAL_WEATHER), # Video - ("function-calling/anthropic-video.py", EVAL_VISION_CAMERA), - ("function-calling/aws-video.py", EVAL_VISION_CAMERA), - ("function-calling/google-video.py", EVAL_VISION_CAMERA), - ("function-calling/moondream-video.py", EVAL_VISION_CAMERA), - ("function-calling/openai-video.py", EVAL_VISION_CAMERA), - ("function-calling/openai-responses-video.py", EVAL_VISION_CAMERA), - ("function-calling/openai-responses-video-http.py", EVAL_VISION_CAMERA), + ("function-calling/function-calling-anthropic-video.py", EVAL_VISION_CAMERA), + ("function-calling/function-calling-aws-video.py", EVAL_VISION_CAMERA), + ("function-calling/function-calling-google-video.py", EVAL_VISION_CAMERA), + ("function-calling/function-calling-moondream-video.py", EVAL_VISION_CAMERA), + ("function-calling/function-calling-openai-video.py", EVAL_VISION_CAMERA), + ("function-calling/function-calling-openai-responses-video.py", EVAL_VISION_CAMERA), + ("function-calling/function-calling-openai-responses-video-http.py", EVAL_VISION_CAMERA), # Currently not working. - # ("function-calling/together.py", EVAL_WEATHER), - # ("function-calling/deepseek.py", EVAL_WEATHER), - # ("function-calling/gemini-openai-format.py", EVAL_WEATHER), + # ("function-calling/function-calling-together.py", EVAL_WEATHER), + # ("function-calling/function-calling-deepseek.py", EVAL_WEATHER), + # ("function-calling/function-calling-gemini-openai-format.py", EVAL_WEATHER), ] TESTS_FEATURES = [ - ("features/switch-languages.py", EVAL_SWITCH_LANGUAGE), - ("features/voicemail-detection.py", EVAL_VOICEMAIL), - ("features/voicemail-detection.py", EVAL_CONVERSATION), - ("features/concurrent-llm-evaluation.py", EVAL_SIMPLE_MATH), + ("features/features-switch-languages.py", EVAL_SWITCH_LANGUAGE), + ("features/features-voicemail-detection.py", EVAL_VOICEMAIL), + ("features/features-voicemail-detection.py", EVAL_CONVERSATION), + ("features/features-concurrent-llm-evaluation.py", EVAL_SIMPLE_MATH), ] TESTS_REALTIME = [ - ("realtime/openai.py", EVAL_WEATHER), - ("realtime/openai-beta.py", EVAL_WEATHER), + ("realtime/realtime-openai.py", EVAL_WEATHER), + ("realtime/realtime-openai-beta.py", EVAL_WEATHER), # OpenAI Realtime not released on Azure yet - # ("realtime/azure.py", EVAL_WEATHER), - ("realtime/azure-beta.py", EVAL_WEATHER), - ("realtime/openai-text.py", EVAL_WEATHER), - ("realtime/openai-beta-text.py", EVAL_WEATHER), - ("realtime/openai-live-video.py", EVAL_VISION_CAMERA), - ("realtime/gemini-live.py", EVAL_SIMPLE_MATH), - ("realtime/gemini-live-local-vad.py", EVAL_SIMPLE_MATH), - ("realtime/gemini-live-function-calling.py", EVAL_WEATHER), - ("realtime/gemini-live-video.py", EVAL_VISION_CAMERA), - ("realtime/gemini-live-google-search.py", EVAL_ONLINE_SEARCH), - ("realtime/gemini-live-vertex-function-calling.py", EVAL_WEATHER), + # ("realtime/realtime-azure.py", EVAL_WEATHER), + ("realtime/realtime-azure-beta.py", EVAL_WEATHER), + ("realtime/realtime-openai-text.py", EVAL_WEATHER), + ("realtime/realtime-openai-beta-text.py", EVAL_WEATHER), + ("realtime/realtime-openai-live-video.py", EVAL_VISION_CAMERA), + ("realtime/realtime-gemini-live.py", EVAL_SIMPLE_MATH), + ("realtime/realtime-gemini-live-local-vad.py", EVAL_SIMPLE_MATH), + ("realtime/realtime-gemini-live-function-calling.py", EVAL_WEATHER), + ("realtime/realtime-gemini-live-video.py", EVAL_VISION_CAMERA), + ("realtime/realtime-gemini-live-google-search.py", EVAL_ONLINE_SEARCH), + ("realtime/realtime-gemini-live-vertex-function-calling.py", EVAL_WEATHER), # Currently not working. - # ("realtime/gemini-live-text.py", EVAL_SIMPLE_MATH), - ("realtime/aws-nova-sonic.py", EVAL_SIMPLE_MATH), - ("realtime/ultravox.py", EVAL_ORDER), - ("realtime/grok.py", EVAL_WEATHER), + # ("realtime/realtime-gemini-live-text.py", EVAL_SIMPLE_MATH), + ("realtime/realtime-aws-nova-sonic.py", EVAL_SIMPLE_MATH), + ("realtime/realtime-ultravox.py", EVAL_ORDER), + ("realtime/realtime-grok.py", EVAL_WEATHER), ] TESTS_VIDEO_AVATAR = [ - ("video-avatar/tavus-video-service.py", EVAL_SIMPLE_MATH), - ("video-avatar/heygen-video-service.py", EVAL_SIMPLE_MATH), - ("video-avatar/simli-layer.py", EVAL_SIMPLE_MATH), - ("video-avatar/lemonslice-transport.py", EVAL_SIMPLE_MATH), + ("video-avatar/video-avatar-tavus-video-service.py", EVAL_SIMPLE_MATH), + ("video-avatar/video-avatar-heygen-video-service.py", EVAL_SIMPLE_MATH), + ("video-avatar/video-avatar-simli-layer.py", EVAL_SIMPLE_MATH), + ("video-avatar/video-avatar-lemonslice-transport.py", EVAL_SIMPLE_MATH), ] TESTS_TURN_MANAGEMENT = [ - ("turn-management/filter-incomplete-turns.py", EVAL_COMPLETE_TURN), + ("turn-management/turn-management-filter-incomplete-turns.py", EVAL_COMPLETE_TURN), ] -TESTS_THINKING_AND_MCP = [ - ("thinking-and-mcp/thinking-anthropic.py", EVAL_SIMPLE_MATH), - ("thinking-and-mcp/thinking-google.py", EVAL_SIMPLE_MATH), - ("thinking-and-mcp/thinking-functions-anthropic.py", EVAL_FLIGHT_STATUS), - ("thinking-and-mcp/thinking-functions-google.py", EVAL_FLIGHT_STATUS), +TESTS_THINKING = [ + ("thinking/thinking-anthropic.py", EVAL_SIMPLE_MATH), + ("thinking/thinking-google.py", EVAL_SIMPLE_MATH), + ("thinking/thinking-functions-anthropic.py", EVAL_FLIGHT_STATUS), + ("thinking/thinking-functions-google.py", EVAL_FLIGHT_STATUS), ] TESTS = [ @@ -264,7 +264,7 @@ TESTS = [ *TESTS_REALTIME, *TESTS_VIDEO_AVATAR, *TESTS_TURN_MANAGEMENT, - *TESTS_THINKING_AND_MCP, + *TESTS_THINKING, ]