wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rename services/ to voice/ and function-calling/, flatten to top level

Browse Source 
Replace the nested services/speech/ and services/function-calling/ with
top-level voice/ and function-calling/ directories. Update eval script
paths and README to match.
This commit is contained in:
Mark Backman
2026-03-31 15:12:06 -04:00
parent f14638a1fd
commit 47b41a0ff7
93 changed files with 96 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
scripts/evals/run-release-evals.py
View File

@@ -96,61 +96,61 @@ EVAL_COMPLETE_TURN = EvalConfig(
)
TESTS_SPEECH = [
("services/speech/cartesia.py", EVAL_SIMPLE_MATH),
("services/speech/cartesia-http.py", EVAL_SIMPLE_MATH),
("services/speech/speechmatics.py", EVAL_SIMPLE_MATH),
("services/speech/speechmatics-vad.py", EVAL_SIMPLE_MATH),
("services/speech/langchain.py", EVAL_SIMPLE_MATH),
("services/speech/deepgram.py", EVAL_SIMPLE_MATH),
("services/speech/deepgram-flux.py", EVAL_SIMPLE_MATH),
("services/speech/deepgram-http.py", EVAL_SIMPLE_MATH),
("services/speech/elevenlabs.py", EVAL_SIMPLE_MATH),
("services/speech/elevenlabs-http.py", EVAL_SIMPLE_MATH),
("services/speech/xai.py", EVAL_SIMPLE_MATH),
("services/speech/azure.py", EVAL_SIMPLE_MATH),
("services/speech/azure-http.py", EVAL_SIMPLE_MATH),
("services/speech/openai.py", EVAL_SIMPLE_MATH),
("services/speech/openai-http.py", EVAL_SIMPLE_MATH),
("services/speech/gladia.py", EVAL_SIMPLE_MATH),
("services/speech/gladia-vad.py", EVAL_SIMPLE_MATH),
("services/speech/lmnt.py", EVAL_SIMPLE_MATH),
("services/speech/groq.py", EVAL_SIMPLE_MATH),
("services/speech/aws.py", EVAL_SIMPLE_MATH),
("services/speech/aws-strands.py", EVAL_WEATHER),
("services/speech/google-gemini-tts.py", EVAL_SIMPLE_MATH),
("services/speech/google.py", EVAL_SIMPLE_MATH),
("services/speech/google-http.py", EVAL_SIMPLE_MATH),
("services/speech/assemblyai.py", EVAL_SIMPLE_MATH),
("services/speech/krisp-viva.py", EVAL_SIMPLE_MATH),
("services/speech/rime.py", EVAL_SIMPLE_MATH),
("services/speech/rime-http.py", EVAL_SIMPLE_MATH),
("services/speech/nvidia.py", EVAL_SIMPLE_MATH),
("services/speech/google-audio-in.py", EVAL_SIMPLE_MATH),
("services/speech/fish.py", EVAL_SIMPLE_MATH),
("services/speech/neuphonic.py", EVAL_SIMPLE_MATH),
("services/speech/neuphonic-http.py", EVAL_SIMPLE_MATH),
("services/speech/fal.py", EVAL_SIMPLE_MATH),
("services/speech/minimax.py", EVAL_SIMPLE_MATH),
("services/speech/sarvam.py", EVAL_SIMPLE_MATH),
("services/speech/sarvam-http.py", EVAL_SIMPLE_MATH),
("services/speech/soniox.py", EVAL_SIMPLE_MATH),
("services/speech/inworld.py", EVAL_SIMPLE_MATH),
("services/speech/inworld-http.py", EVAL_SIMPLE_MATH),
("services/speech/asyncai.py", EVAL_SIMPLE_MATH),
("services/speech/asyncai-http.py", EVAL_SIMPLE_MATH),
("services/speech/aicoustics.py", EVAL_SIMPLE_MATH),
("services/speech/hume.py", EVAL_SIMPLE_MATH),
("services/speech/gradium.py", EVAL_SIMPLE_MATH),
("services/speech/camb.py", EVAL_SIMPLE_MATH),
("services/speech/piper.py", EVAL_SIMPLE_MATH),
("services/speech/kokoro.py", EVAL_SIMPLE_MATH),
("services/speech/resemble.py", EVAL_SIMPLE_MATH),
("services/speech/smallest.py", EVAL_SIMPLE_MATH),
("services/speech/openai-responses.py", EVAL_SIMPLE_MATH),
("services/speech/openai-responses-http.py", EVAL_SIMPLE_MATH),
TESTS_VOICE = [
("voice/cartesia.py", EVAL_SIMPLE_MATH),
("voice/cartesia-http.py", EVAL_SIMPLE_MATH),
("voice/speechmatics.py", EVAL_SIMPLE_MATH),
("voice/speechmatics-vad.py", EVAL_SIMPLE_MATH),
("voice/langchain.py", EVAL_SIMPLE_MATH),
("voice/deepgram.py", EVAL_SIMPLE_MATH),
("voice/deepgram-flux.py", EVAL_SIMPLE_MATH),
("voice/deepgram-http.py", EVAL_SIMPLE_MATH),
("voice/elevenlabs.py", EVAL_SIMPLE_MATH),
("voice/elevenlabs-http.py", EVAL_SIMPLE_MATH),
("voice/xai.py", EVAL_SIMPLE_MATH),
("voice/azure.py", EVAL_SIMPLE_MATH),
("voice/azure-http.py", EVAL_SIMPLE_MATH),
("voice/openai.py", EVAL_SIMPLE_MATH),
("voice/openai-http.py", EVAL_SIMPLE_MATH),
("voice/gladia.py", EVAL_SIMPLE_MATH),
("voice/gladia-vad.py", EVAL_SIMPLE_MATH),
("voice/lmnt.py", EVAL_SIMPLE_MATH),
("voice/groq.py", EVAL_SIMPLE_MATH),
("voice/aws.py", EVAL_SIMPLE_MATH),
("voice/aws-strands.py", EVAL_WEATHER),
("voice/google-gemini-tts.py", EVAL_SIMPLE_MATH),
("voice/google.py", EVAL_SIMPLE_MATH),
("voice/google-http.py", EVAL_SIMPLE_MATH),
("voice/assemblyai.py", EVAL_SIMPLE_MATH),
("voice/krisp-viva.py", EVAL_SIMPLE_MATH),
("voice/rime.py", EVAL_SIMPLE_MATH),
("voice/rime-http.py", EVAL_SIMPLE_MATH),
("voice/nvidia.py", EVAL_SIMPLE_MATH),
("voice/google-audio-in.py", EVAL_SIMPLE_MATH),
("voice/fish.py", EVAL_SIMPLE_MATH),
("voice/neuphonic.py", EVAL_SIMPLE_MATH),
("voice/neuphonic-http.py", EVAL_SIMPLE_MATH),
("voice/fal.py", EVAL_SIMPLE_MATH),
("voice/minimax.py", EVAL_SIMPLE_MATH),
("voice/sarvam.py", EVAL_SIMPLE_MATH),
("voice/sarvam-http.py", EVAL_SIMPLE_MATH),
("voice/soniox.py", EVAL_SIMPLE_MATH),
("voice/inworld.py", EVAL_SIMPLE_MATH),
("voice/inworld-http.py", EVAL_SIMPLE_MATH),
("voice/asyncai.py", EVAL_SIMPLE_MATH),
("voice/asyncai-http.py", EVAL_SIMPLE_MATH),
("voice/aicoustics.py", EVAL_SIMPLE_MATH),
("voice/hume.py", EVAL_SIMPLE_MATH),
("voice/gradium.py", EVAL_SIMPLE_MATH),
("voice/camb.py", EVAL_SIMPLE_MATH),
("voice/piper.py", EVAL_SIMPLE_MATH),
("voice/kokoro.py", EVAL_SIMPLE_MATH),
("voice/resemble.py", EVAL_SIMPLE_MATH),
("voice/smallest.py", EVAL_SIMPLE_MATH),
("voice/openai-responses.py", EVAL_SIMPLE_MATH),
("voice/openai-responses-http.py", EVAL_SIMPLE_MATH),
# Needs a local XTTS docker instance running.
# ("services/speech/xtts.py", EVAL_SIMPLE_MATH),
# ("voice/xtts.py", EVAL_SIMPLE_MATH),
]
TESTS_VISION = [
@@ -169,44 +169,44 @@ TESTS_VISION = [
TESTS_FUNCTION_CALLING = [
("getting-started/07-function-calling.py", EVAL_WEATHER),
("getting-started/07-function-calling.py", EVAL_WEATHER_AND_RESTAURANT),
("services/function-calling/openai-responses.py", EVAL_WEATHER),
("services/function-calling/openai-responses.py", EVAL_WEATHER_AND_RESTAURANT),
("services/function-calling/openai-responses-http.py", EVAL_WEATHER),
("services/function-calling/openai-responses-http.py", EVAL_WEATHER_AND_RESTAURANT),
("services/function-calling/anthropic.py", EVAL_WEATHER),
("services/function-calling/anthropic.py", EVAL_WEATHER_AND_RESTAURANT),
("services/function-calling/openai.py", EVAL_WEATHER),
("services/function-calling/google.py", EVAL_WEATHER),
("services/function-calling/google.py", EVAL_WEATHER_AND_RESTAURANT),
("services/function-calling/groq.py", EVAL_WEATHER),
("services/function-calling/grok.py", EVAL_WEATHER),
("services/function-calling/azure.py", EVAL_WEATHER),
("services/function-calling/fireworks.py", EVAL_WEATHER),
("services/function-calling/nvidia.py", EVAL_WEATHER),
("services/function-calling/cerebras.py", EVAL_WEATHER),
("services/function-calling/openrouter.py", EVAL_WEATHER),
("services/function-calling/perplexity.py", EVAL_WEATHER),
("services/function-calling/google-vertex.py", EVAL_WEATHER),
("services/function-calling/qwen.py", EVAL_WEATHER),
("services/function-calling/aws.py", EVAL_WEATHER),
("services/function-calling/sambanova.py", EVAL_WEATHER),
("services/function-calling/aws.py", EVAL_WEATHER_AND_RESTAURANT),
("services/function-calling/nebius.py", EVAL_WEATHER),
("services/function-calling/mistral.py", EVAL_WEATHER),
("services/function-calling/sarvam.py", EVAL_WEATHER),
("services/function-calling/novita.py", EVAL_WEATHER),
("function-calling/openai-responses.py", EVAL_WEATHER),
("function-calling/openai-responses.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/openai-responses-http.py", EVAL_WEATHER),
("function-calling/openai-responses-http.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/anthropic.py", EVAL_WEATHER),
("function-calling/anthropic.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/openai.py", EVAL_WEATHER),
("function-calling/google.py", EVAL_WEATHER),
("function-calling/google.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/groq.py", EVAL_WEATHER),
("function-calling/grok.py", EVAL_WEATHER),
("function-calling/azure.py", EVAL_WEATHER),
("function-calling/fireworks.py", EVAL_WEATHER),
("function-calling/nvidia.py", EVAL_WEATHER),
("function-calling/cerebras.py", EVAL_WEATHER),
("function-calling/openrouter.py", EVAL_WEATHER),
("function-calling/perplexity.py", EVAL_WEATHER),
("function-calling/google-vertex.py", EVAL_WEATHER),
("function-calling/qwen.py", EVAL_WEATHER),
("function-calling/aws.py", EVAL_WEATHER),
("function-calling/sambanova.py", EVAL_WEATHER),
("function-calling/aws.py", EVAL_WEATHER_AND_RESTAURANT),
("function-calling/nebius.py", EVAL_WEATHER),
("function-calling/mistral.py", EVAL_WEATHER),
("function-calling/sarvam.py", EVAL_WEATHER),
("function-calling/novita.py", EVAL_WEATHER),
# Video
("services/function-calling/anthropic-video.py", EVAL_VISION_CAMERA),
("services/function-calling/aws-video.py", EVAL_VISION_CAMERA),
("services/function-calling/google-video.py", EVAL_VISION_CAMERA),
("services/function-calling/moondream-video.py", EVAL_VISION_CAMERA),
("services/function-calling/openai-video.py", EVAL_VISION_CAMERA),
("services/function-calling/openai-responses-video.py", EVAL_VISION_CAMERA),
("services/function-calling/openai-responses-video-http.py", EVAL_VISION_CAMERA),
("function-calling/anthropic-video.py", EVAL_VISION_CAMERA),
("function-calling/aws-video.py", EVAL_VISION_CAMERA),
("function-calling/google-video.py", EVAL_VISION_CAMERA),
("function-calling/moondream-video.py", EVAL_VISION_CAMERA),
("function-calling/openai-video.py", EVAL_VISION_CAMERA),
("function-calling/openai-responses-video.py", EVAL_VISION_CAMERA),
("function-calling/openai-responses-video-http.py", EVAL_VISION_CAMERA),
# Currently not working.
# ("services/function-calling/together.py", EVAL_WEATHER),
# ("services/function-calling/deepseek.py", EVAL_WEATHER),
# ("services/function-calling/gemini-openai-format.py", EVAL_WEATHER),
# ("function-calling/together.py", EVAL_WEATHER),
# ("function-calling/deepseek.py", EVAL_WEATHER),
# ("function-calling/gemini-openai-format.py", EVAL_WEATHER),
]
TESTS_FEATURES = [
@@ -257,7 +257,7 @@ TESTS_THINKING_AND_MCP = [
]
TESTS = [
*TESTS_SPEECH,
*TESTS_VOICE,
*TESTS_VISION,
*TESTS_FUNCTION_CALLING,
*TESTS_FEATURES,