wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update eval script paths to match renamed example files

Browse Source
This commit is contained in:
Mark Backman
2026-03-31 22:09:42 -04:00
parent d3021b4590
commit 080da8b94c
1 changed files with 128 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
scripts/evals/run-release-evals.py
View File

@@ -97,70 +97,70 @@ EVAL_COMPLETE_TURN = EvalConfig(
TESTS_VOICE = [
("voice/cartesia.py", EVAL_SIMPLE_MATH),
("voice/cartesia-http.py", EVAL_SIMPLE_MATH),
("voice/speechmatics.py", EVAL_SIMPLE_MATH),
("voice/speechmatics-vad.py", EVAL_SIMPLE_MATH),
("voice/langchain.py", EVAL_SIMPLE_MATH),
("voice/deepgram.py", EVAL_SIMPLE_MATH),
("voice/deepgram-flux.py", EVAL_SIMPLE_MATH),
("voice/deepgram-http.py", EVAL_SIMPLE_MATH),
("voice/elevenlabs.py", EVAL_SIMPLE_MATH),
("voice/elevenlabs-http.py", EVAL_SIMPLE_MATH),
("voice/xai.py", EVAL_SIMPLE_MATH),
("voice/azure.py", EVAL_SIMPLE_MATH),
("voice/azure-http.py", EVAL_SIMPLE_MATH),
("voice/openai.py", EVAL_SIMPLE_MATH),
("voice/openai-http.py", EVAL_SIMPLE_MATH),
("voice/gladia.py", EVAL_SIMPLE_MATH),
("voice/gladia-vad.py", EVAL_SIMPLE_MATH),
("voice/lmnt.py", EVAL_SIMPLE_MATH),
("voice/groq.py", EVAL_SIMPLE_MATH),
("voice/aws.py", EVAL_SIMPLE_MATH),
("voice/aws-strands.py", EVAL_WEATHER),
("voice/google-gemini-tts.py", EVAL_SIMPLE_MATH),
("voice/google.py", EVAL_SIMPLE_MATH),
("voice/google-http.py", EVAL_SIMPLE_MATH),
("voice/assemblyai.py", EVAL_SIMPLE_MATH),
("voice/krisp-viva.py", EVAL_SIMPLE_MATH),
("voice/rime.py", EVAL_SIMPLE_MATH),
("voice/rime-http.py", EVAL_SIMPLE_MATH),
("voice/nvidia.py", EVAL_SIMPLE_MATH),
("voice/google-audio-in.py", EVAL_SIMPLE_MATH),
("voice/fish.py", EVAL_SIMPLE_MATH),
("voice/neuphonic.py", EVAL_SIMPLE_MATH),
("voice/neuphonic-http.py", EVAL_SIMPLE_MATH),
("voice/fal.py", EVAL_SIMPLE_MATH),
("voice/minimax.py", EVAL_SIMPLE_MATH),
("voice/sarvam.py", EVAL_SIMPLE_MATH),
("voice/sarvam-http.py", EVAL_SIMPLE_MATH),
("voice/soniox.py", EVAL_SIMPLE_MATH),
("voice/inworld.py", EVAL_SIMPLE_MATH),
("voice/inworld-http.py", EVAL_SIMPLE_MATH),
("voice/asyncai.py", EVAL_SIMPLE_MATH),
("voice/asyncai-http.py", EVAL_SIMPLE_MATH),
("voice/aicoustics.py", EVAL_SIMPLE_MATH),
("voice/hume.py", EVAL_SIMPLE_MATH),
("voice/gradium.py", EVAL_SIMPLE_MATH),
("voice/camb.py", EVAL_SIMPLE_MATH),
("voice/piper.py", EVAL_SIMPLE_MATH),
("voice/kokoro.py", EVAL_SIMPLE_MATH),
("voice/resemble.py", EVAL_SIMPLE_MATH),
("voice/smallest.py", EVAL_SIMPLE_MATH),
("voice/openai-responses.py", EVAL_SIMPLE_MATH),
("voice/openai-responses-http.py", EVAL_SIMPLE_MATH),
("voice/voice-cartesia.py", EVAL_SIMPLE_MATH),
("voice/voice-cartesia-http.py", EVAL_SIMPLE_MATH),
("voice/voice-speechmatics.py", EVAL_SIMPLE_MATH),
("voice/voice-speechmatics-vad.py", EVAL_SIMPLE_MATH),
("voice/voice-langchain.py", EVAL_SIMPLE_MATH),
("voice/voice-deepgram.py", EVAL_SIMPLE_MATH),
("voice/voice-deepgram-flux.py", EVAL_SIMPLE_MATH),
("voice/voice-deepgram-http.py", EVAL_SIMPLE_MATH),
("voice/voice-elevenlabs.py", EVAL_SIMPLE_MATH),
("voice/voice-elevenlabs-http.py", EVAL_SIMPLE_MATH),
("voice/voice-xai.py", EVAL_SIMPLE_MATH),
("voice/voice-azure.py", EVAL_SIMPLE_MATH),
("voice/voice-azure-http.py", EVAL_SIMPLE_MATH),
("voice/voice-openai.py", EVAL_SIMPLE_MATH),
("voice/voice-openai-http.py", EVAL_SIMPLE_MATH),
("voice/voice-gladia.py", EVAL_SIMPLE_MATH),
("voice/voice-gladia-vad.py", EVAL_SIMPLE_MATH),
("voice/voice-lmnt.py", EVAL_SIMPLE_MATH),
("voice/voice-groq.py", EVAL_SIMPLE_MATH),
("voice/voice-aws.py", EVAL_SIMPLE_MATH),
("voice/voice-aws-strands.py", EVAL_WEATHER),
("voice/voice-google-gemini-tts.py", EVAL_SIMPLE_MATH),
("voice/voice-google.py", EVAL_SIMPLE_MATH),
("voice/voice-google-http.py", EVAL_SIMPLE_MATH),
("voice/voice-assemblyai.py", EVAL_SIMPLE_MATH),
("voice/voice-krisp-viva.py", EVAL_SIMPLE_MATH),
("voice/voice-rime.py", EVAL_SIMPLE_MATH),
("voice/voice-rime-http.py", EVAL_SIMPLE_MATH),
("voice/voice-nvidia.py", EVAL_SIMPLE_MATH),
("voice/voice-google-audio-in.py", EVAL_SIMPLE_MATH),
("voice/voice-fish.py", EVAL_SIMPLE_MATH),
("voice/voice-neuphonic.py", EVAL_SIMPLE_MATH),
("voice/voice-neuphonic-http.py", EVAL_SIMPLE_MATH),
("voice/voice-fal.py", EVAL_SIMPLE_MATH),
("voice/voice-minimax.py", EVAL_SIMPLE_MATH),
("voice/voice-sarvam.py", EVAL_SIMPLE_MATH),
("voice/voice-sarvam-http.py", EVAL_SIMPLE_MATH),
("voice/voice-soniox.py", EVAL_SIMPLE_MATH),
("voice/voice-inworld.py", EVAL_SIMPLE_MATH),
("voice/voice-inworld-http.py", EVAL_SIMPLE_MATH),
("voice/voice-asyncai.py", EVAL_SIMPLE_MATH),
("voice/voice-asyncai-http.py", EVAL_SIMPLE_MATH),
("voice/voice-aicoustics.py", EVAL_SIMPLE_MATH),
("voice/voice-hume.py", EVAL_SIMPLE_MATH),
("voice/voice-gradium.py", EVAL_SIMPLE_MATH),
("voice/voice-camb.py", EVAL_SIMPLE_MATH),
("voice/voice-piper.py", EVAL_SIMPLE_MATH),
("voice/voice-kokoro.py", EVAL_SIMPLE_MATH),
("voice/voice-resemble.py", EVAL_SIMPLE_MATH),
("voice/voice-smallest.py", EVAL_SIMPLE_MATH),
("voice/voice-openai-responses.py", EVAL_SIMPLE_MATH),
("voice/voice-openai-responses-http.py", EVAL_SIMPLE_MATH),
# Needs a local XTTS docker instance running.
# ("voice/xtts.py", EVAL_SIMPLE_MATH),
# ("voice/voice-xtts.py", EVAL_SIMPLE_MATH),
]
TESTS_VISION = [
("vision/openai.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/openai-responses.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/openai-responses-http.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/anthropic.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/aws.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/gemini-flash.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/moondream.py", EVAL_VISION_IMAGE()),
("vision/vision-openai.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/vision-openai-responses.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/vision-openai-responses-http.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/vision-anthropic.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/vision-aws.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/vision-gemini-flash.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
("vision/vision-moondream.py", EVAL_VISION_IMAGE()),
]
# For a few major services, we also test parallel function calling.
@@ -169,91 +169,91 @@ TESTS_VISION = [
TESTS_FUNCTION_CALLING = [
("getting-started/07-function-calling.py", EVAL_WEATHER),
("getting-started/07-function-calling.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/openai-responses.py", EVAL_WEATHER),
("function-calling/openai-responses.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/openai-responses-http.py", EVAL_WEATHER),
("function-calling/openai-responses-http.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/anthropic.py", EVAL_WEATHER),
("function-calling/anthropic.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/openai.py", EVAL_WEATHER),
("function-calling/google.py", EVAL_WEATHER),
("function-calling/google.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/groq.py", EVAL_WEATHER),
("function-calling/grok.py", EVAL_WEATHER),
("function-calling/azure.py", EVAL_WEATHER),
("function-calling/fireworks.py", EVAL_WEATHER),
("function-calling/nvidia.py", EVAL_WEATHER),
("function-calling/cerebras.py", EVAL_WEATHER),
("function-calling/openrouter.py", EVAL_WEATHER),
("function-calling/perplexity.py", EVAL_WEATHER),
("function-calling/google-vertex.py", EVAL_WEATHER),
("function-calling/qwen.py", EVAL_WEATHER),
("function-calling/aws.py", EVAL_WEATHER),
("function-calling/sambanova.py", EVAL_WEATHER),
("function-calling/aws.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/nebius.py", EVAL_WEATHER),
("function-calling/mistral.py", EVAL_WEATHER),
("function-calling/sarvam.py", EVAL_WEATHER),
("function-calling/novita.py", EVAL_WEATHER),
("function-calling/function-calling-openai-responses.py", EVAL_WEATHER),
("function-calling/function-calling-openai-responses.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/function-calling-openai-responses-http.py", EVAL_WEATHER),
("function-calling/function-calling-openai-responses-http.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/function-calling-anthropic.py", EVAL_WEATHER),
("function-calling/function-calling-anthropic.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/function-calling-openai.py", EVAL_WEATHER),
("function-calling/function-calling-google.py", EVAL_WEATHER),
("function-calling/function-calling-google.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/function-calling-groq.py", EVAL_WEATHER),
("function-calling/function-calling-grok.py", EVAL_WEATHER),
("function-calling/function-calling-azure.py", EVAL_WEATHER),
("function-calling/function-calling-fireworks.py", EVAL_WEATHER),
("function-calling/function-calling-nvidia.py", EVAL_WEATHER),
("function-calling/function-calling-cerebras.py", EVAL_WEATHER),
("function-calling/function-calling-openrouter.py", EVAL_WEATHER),
("function-calling/function-calling-perplexity.py", EVAL_WEATHER),
("function-calling/function-calling-google-vertex.py", EVAL_WEATHER),
("function-calling/function-calling-qwen.py", EVAL_WEATHER),
("function-calling/function-calling-aws.py", EVAL_WEATHER),
("function-calling/function-calling-sambanova.py", EVAL_WEATHER),
("function-calling/function-calling-aws.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/function-calling-nebius.py", EVAL_WEATHER),
("function-calling/function-calling-mistral.py", EVAL_WEATHER),
("function-calling/function-calling-sarvam.py", EVAL_WEATHER),
("function-calling/function-calling-novita.py", EVAL_WEATHER),
# Video
("function-calling/anthropic-video.py", EVAL_VISION_CAMERA),
("function-calling/aws-video.py", EVAL_VISION_CAMERA),
("function-calling/google-video.py", EVAL_VISION_CAMERA),
("function-calling/moondream-video.py", EVAL_VISION_CAMERA),
("function-calling/openai-video.py", EVAL_VISION_CAMERA),
("function-calling/openai-responses-video.py", EVAL_VISION_CAMERA),
("function-calling/openai-responses-video-http.py", EVAL_VISION_CAMERA),
("function-calling/function-calling-anthropic-video.py", EVAL_VISION_CAMERA),
("function-calling/function-calling-aws-video.py", EVAL_VISION_CAMERA),
("function-calling/function-calling-google-video.py", EVAL_VISION_CAMERA),
("function-calling/function-calling-moondream-video.py", EVAL_VISION_CAMERA),
("function-calling/function-calling-openai-video.py", EVAL_VISION_CAMERA),
("function-calling/function-calling-openai-responses-video.py", EVAL_VISION_CAMERA),
("function-calling/function-calling-openai-responses-video-http.py", EVAL_VISION_CAMERA),
# Currently not working.
# ("function-calling/together.py", EVAL_WEATHER),
# ("function-calling/deepseek.py", EVAL_WEATHER),
# ("function-calling/gemini-openai-format.py", EVAL_WEATHER),
# ("function-calling/function-calling-together.py", EVAL_WEATHER),
# ("function-calling/function-calling-deepseek.py", EVAL_WEATHER),
# ("function-calling/function-calling-gemini-openai-format.py", EVAL_WEATHER),
]
TESTS_FEATURES = [
("features/switch-languages.py", EVAL_SWITCH_LANGUAGE),
("features/voicemail-detection.py", EVAL_VOICEMAIL),
("features/voicemail-detection.py", EVAL_CONVERSATION),
("features/concurrent-llm-evaluation.py", EVAL_SIMPLE_MATH),
("features/features-switch-languages.py", EVAL_SWITCH_LANGUAGE),
("features/features-voicemail-detection.py", EVAL_VOICEMAIL),
("features/features-voicemail-detection.py", EVAL_CONVERSATION),
("features/features-concurrent-llm-evaluation.py", EVAL_SIMPLE_MATH),
]
TESTS_REALTIME = [
("realtime/openai.py", EVAL_WEATHER),
("realtime/openai-beta.py", EVAL_WEATHER),
("realtime/realtime-openai.py", EVAL_WEATHER),
("realtime/realtime-openai-beta.py", EVAL_WEATHER),
# OpenAI Realtime not released on Azure yet
# ("realtime/azure.py", EVAL_WEATHER),
("realtime/azure-beta.py", EVAL_WEATHER),
("realtime/openai-text.py", EVAL_WEATHER),
("realtime/openai-beta-text.py", EVAL_WEATHER),
("realtime/openai-live-video.py", EVAL_VISION_CAMERA),
("realtime/gemini-live.py", EVAL_SIMPLE_MATH),
("realtime/gemini-live-local-vad.py", EVAL_SIMPLE_MATH),
("realtime/gemini-live-function-calling.py", EVAL_WEATHER),
("realtime/gemini-live-video.py", EVAL_VISION_CAMERA),
("realtime/gemini-live-google-search.py", EVAL_ONLINE_SEARCH),
("realtime/gemini-live-vertex-function-calling.py", EVAL_WEATHER),
# ("realtime/realtime-azure.py", EVAL_WEATHER),
("realtime/realtime-azure-beta.py", EVAL_WEATHER),
("realtime/realtime-openai-text.py", EVAL_WEATHER),
("realtime/realtime-openai-beta-text.py", EVAL_WEATHER),
("realtime/realtime-openai-live-video.py", EVAL_VISION_CAMERA),
("realtime/realtime-gemini-live.py", EVAL_SIMPLE_MATH),
("realtime/realtime-gemini-live-local-vad.py", EVAL_SIMPLE_MATH),
("realtime/realtime-gemini-live-function-calling.py", EVAL_WEATHER),
("realtime/realtime-gemini-live-video.py", EVAL_VISION_CAMERA),
("realtime/realtime-gemini-live-google-search.py", EVAL_ONLINE_SEARCH),
("realtime/realtime-gemini-live-vertex-function-calling.py", EVAL_WEATHER),
# Currently not working.
# ("realtime/gemini-live-text.py", EVAL_SIMPLE_MATH),
("realtime/aws-nova-sonic.py", EVAL_SIMPLE_MATH),
("realtime/ultravox.py", EVAL_ORDER),
("realtime/grok.py", EVAL_WEATHER),
# ("realtime/realtime-gemini-live-text.py", EVAL_SIMPLE_MATH),
("realtime/realtime-aws-nova-sonic.py", EVAL_SIMPLE_MATH),
("realtime/realtime-ultravox.py", EVAL_ORDER),
("realtime/realtime-grok.py", EVAL_WEATHER),
]
TESTS_VIDEO_AVATAR = [
("video-avatar/tavus-video-service.py", EVAL_SIMPLE_MATH),
("video-avatar/heygen-video-service.py", EVAL_SIMPLE_MATH),
("video-avatar/simli-layer.py", EVAL_SIMPLE_MATH),
("video-avatar/lemonslice-transport.py", EVAL_SIMPLE_MATH),
("video-avatar/video-avatar-tavus-video-service.py", EVAL_SIMPLE_MATH),
("video-avatar/video-avatar-heygen-video-service.py", EVAL_SIMPLE_MATH),
("video-avatar/video-avatar-simli-layer.py", EVAL_SIMPLE_MATH),
("video-avatar/video-avatar-lemonslice-transport.py", EVAL_SIMPLE_MATH),
]
TESTS_TURN_MANAGEMENT = [
("turn-management/filter-incomplete-turns.py", EVAL_COMPLETE_TURN),
("turn-management/turn-management-filter-incomplete-turns.py", EVAL_COMPLETE_TURN),
]
TESTS_THINKING_AND_MCP = [
("thinking-and-mcp/thinking-anthropic.py", EVAL_SIMPLE_MATH),
("thinking-and-mcp/thinking-google.py", EVAL_SIMPLE_MATH),
("thinking-and-mcp/thinking-functions-anthropic.py", EVAL_FLIGHT_STATUS),
("thinking-and-mcp/thinking-functions-google.py", EVAL_FLIGHT_STATUS),
TESTS_THINKING = [
("thinking/thinking-anthropic.py", EVAL_SIMPLE_MATH),
("thinking/thinking-google.py", EVAL_SIMPLE_MATH),
("thinking/thinking-functions-anthropic.py", EVAL_FLIGHT_STATUS),
("thinking/thinking-functions-google.py", EVAL_FLIGHT_STATUS),
]
TESTS = [
@@ -264,7 +264,7 @@ TESTS = [
*TESTS_REALTIME,
*TESTS_VIDEO_AVATAR,
*TESTS_TURN_MANAGEMENT,
*TESTS_THINKING_AND_MCP,
*TESTS_THINKING,
]