diff --git a/examples/README.md b/examples/README.md index 08dc58de5..a7fabaa73 100644 --- a/examples/README.md +++ b/examples/README.md @@ -63,10 +63,11 @@ Progressive introduction to Pipecat, from minimal TTS to a full voice agent with ### [`services/`](./services/) -Service provider integration examples, organized into subfolders: +Full STT + LLM + TTS pipelines showcasing different speech service providers (Deepgram, ElevenLabs, Cartesia, etc.) -- **[`speech/`](./services/speech/)** — Full STT + LLM + TTS pipelines showcasing different speech service providers (Deepgram, ElevenLabs, Cartesia, etc.) -- **[`function-calling/`](./services/function-calling/)** — Function calling with different LLM providers (OpenAI, Anthropic, Google, etc.) +### [`function-calling/`](./function-calling/) + +Function calling with different LLM providers (OpenAI, Anthropic, Google, etc.) ### [`transcription/`](./transcription/) diff --git a/examples/services/function-calling/anthropic-video.py b/examples/function-calling/anthropic-video.py similarity index 100% rename from examples/services/function-calling/anthropic-video.py rename to examples/function-calling/anthropic-video.py diff --git a/examples/services/function-calling/anthropic.py b/examples/function-calling/anthropic.py similarity index 100% rename from examples/services/function-calling/anthropic.py rename to examples/function-calling/anthropic.py diff --git a/examples/services/function-calling/aws-video.py b/examples/function-calling/aws-video.py similarity index 100% rename from examples/services/function-calling/aws-video.py rename to examples/function-calling/aws-video.py diff --git a/examples/services/function-calling/aws.py b/examples/function-calling/aws.py similarity index 100% rename from examples/services/function-calling/aws.py rename to examples/function-calling/aws.py diff --git a/examples/services/function-calling/azure.py b/examples/function-calling/azure.py similarity index 100% rename from examples/services/function-calling/azure.py rename to examples/function-calling/azure.py diff --git a/examples/services/function-calling/cerebras.py b/examples/function-calling/cerebras.py similarity index 100% rename from examples/services/function-calling/cerebras.py rename to examples/function-calling/cerebras.py diff --git a/examples/services/function-calling/deepseek.py b/examples/function-calling/deepseek.py similarity index 100% rename from examples/services/function-calling/deepseek.py rename to examples/function-calling/deepseek.py diff --git a/examples/services/function-calling/direct.py b/examples/function-calling/direct.py similarity index 100% rename from examples/services/function-calling/direct.py rename to examples/function-calling/direct.py diff --git a/examples/services/function-calling/fireworks.py b/examples/function-calling/fireworks.py similarity index 100% rename from examples/services/function-calling/fireworks.py rename to examples/function-calling/fireworks.py diff --git a/examples/services/function-calling/gemini-openai-format.py b/examples/function-calling/gemini-openai-format.py similarity index 100% rename from examples/services/function-calling/gemini-openai-format.py rename to examples/function-calling/gemini-openai-format.py diff --git a/examples/services/function-calling/google-vertex-ai.py b/examples/function-calling/google-vertex-ai.py similarity index 100% rename from examples/services/function-calling/google-vertex-ai.py rename to examples/function-calling/google-vertex-ai.py diff --git a/examples/services/function-calling/google-video.py b/examples/function-calling/google-video.py similarity index 100% rename from examples/services/function-calling/google-video.py rename to examples/function-calling/google-video.py diff --git a/examples/services/function-calling/google.py b/examples/function-calling/google.py similarity index 100% rename from examples/services/function-calling/google.py rename to examples/function-calling/google.py diff --git a/examples/services/function-calling/grok.py b/examples/function-calling/grok.py similarity index 100% rename from examples/services/function-calling/grok.py rename to examples/function-calling/grok.py diff --git a/examples/services/function-calling/groq.py b/examples/function-calling/groq.py similarity index 100% rename from examples/services/function-calling/groq.py rename to examples/function-calling/groq.py diff --git a/examples/services/function-calling/mistral.py b/examples/function-calling/mistral.py similarity index 100% rename from examples/services/function-calling/mistral.py rename to examples/function-calling/mistral.py diff --git a/examples/services/function-calling/moondream-video.py b/examples/function-calling/moondream-video.py similarity index 100% rename from examples/services/function-calling/moondream-video.py rename to examples/function-calling/moondream-video.py diff --git a/examples/services/function-calling/nebius.py b/examples/function-calling/nebius.py similarity index 100% rename from examples/services/function-calling/nebius.py rename to examples/function-calling/nebius.py diff --git a/examples/services/function-calling/novita.py b/examples/function-calling/novita.py similarity index 100% rename from examples/services/function-calling/novita.py rename to examples/function-calling/novita.py diff --git a/examples/services/function-calling/nvidia.py b/examples/function-calling/nvidia.py similarity index 100% rename from examples/services/function-calling/nvidia.py rename to examples/function-calling/nvidia.py diff --git a/examples/services/function-calling/ollama.py b/examples/function-calling/ollama.py similarity index 100% rename from examples/services/function-calling/ollama.py rename to examples/function-calling/ollama.py diff --git a/examples/services/function-calling/openai-responses-http.py b/examples/function-calling/openai-responses-http.py similarity index 100% rename from examples/services/function-calling/openai-responses-http.py rename to examples/function-calling/openai-responses-http.py diff --git a/examples/services/function-calling/openai-responses-video-http.py b/examples/function-calling/openai-responses-video-http.py similarity index 100% rename from examples/services/function-calling/openai-responses-video-http.py rename to examples/function-calling/openai-responses-video-http.py diff --git a/examples/services/function-calling/openai-responses-video.py b/examples/function-calling/openai-responses-video.py similarity index 100% rename from examples/services/function-calling/openai-responses-video.py rename to examples/function-calling/openai-responses-video.py diff --git a/examples/services/function-calling/openai-responses.py b/examples/function-calling/openai-responses.py similarity index 100% rename from examples/services/function-calling/openai-responses.py rename to examples/function-calling/openai-responses.py diff --git a/examples/services/function-calling/openai-video.py b/examples/function-calling/openai-video.py similarity index 100% rename from examples/services/function-calling/openai-video.py rename to examples/function-calling/openai-video.py diff --git a/examples/services/function-calling/openai.py b/examples/function-calling/openai.py similarity index 100% rename from examples/services/function-calling/openai.py rename to examples/function-calling/openai.py diff --git a/examples/services/function-calling/openrouter.py b/examples/function-calling/openrouter.py similarity index 100% rename from examples/services/function-calling/openrouter.py rename to examples/function-calling/openrouter.py diff --git a/examples/services/function-calling/perplexity.py b/examples/function-calling/perplexity.py similarity index 100% rename from examples/services/function-calling/perplexity.py rename to examples/function-calling/perplexity.py diff --git a/examples/services/function-calling/qwen.py b/examples/function-calling/qwen.py similarity index 100% rename from examples/services/function-calling/qwen.py rename to examples/function-calling/qwen.py diff --git a/examples/services/function-calling/sambanova.py b/examples/function-calling/sambanova.py similarity index 100% rename from examples/services/function-calling/sambanova.py rename to examples/function-calling/sambanova.py diff --git a/examples/services/function-calling/sarvam.py b/examples/function-calling/sarvam.py similarity index 100% rename from examples/services/function-calling/sarvam.py rename to examples/function-calling/sarvam.py diff --git a/examples/services/function-calling/together.py b/examples/function-calling/together.py similarity index 100% rename from examples/services/function-calling/together.py rename to examples/function-calling/together.py diff --git a/examples/services/speech/aicoustics.py b/examples/services/aicoustics.py similarity index 100% rename from examples/services/speech/aicoustics.py rename to examples/services/aicoustics.py diff --git a/examples/services/speech/assemblyai-turn-detection.py b/examples/services/assemblyai-turn-detection.py similarity index 100% rename from examples/services/speech/assemblyai-turn-detection.py rename to examples/services/assemblyai-turn-detection.py diff --git a/examples/services/speech/assemblyai.py b/examples/services/assemblyai.py similarity index 100% rename from examples/services/speech/assemblyai.py rename to examples/services/assemblyai.py diff --git a/examples/services/speech/asyncai-http.py b/examples/services/asyncai-http.py similarity index 100% rename from examples/services/speech/asyncai-http.py rename to examples/services/asyncai-http.py diff --git a/examples/services/speech/asyncai.py b/examples/services/asyncai.py similarity index 100% rename from examples/services/speech/asyncai.py rename to examples/services/asyncai.py diff --git a/examples/services/speech/aws-strands.py b/examples/services/aws-strands.py similarity index 100% rename from examples/services/speech/aws-strands.py rename to examples/services/aws-strands.py diff --git a/examples/services/speech/aws.py b/examples/services/aws.py similarity index 100% rename from examples/services/speech/aws.py rename to examples/services/aws.py diff --git a/examples/services/speech/azure-http.py b/examples/services/azure-http.py similarity index 100% rename from examples/services/speech/azure-http.py rename to examples/services/azure-http.py diff --git a/examples/services/speech/azure.py b/examples/services/azure.py similarity index 100% rename from examples/services/speech/azure.py rename to examples/services/azure.py diff --git a/examples/services/speech/camb.py b/examples/services/camb.py similarity index 100% rename from examples/services/speech/camb.py rename to examples/services/camb.py diff --git a/examples/services/speech/cartesia-http.py b/examples/services/cartesia-http.py similarity index 100% rename from examples/services/speech/cartesia-http.py rename to examples/services/cartesia-http.py diff --git a/examples/services/speech/cartesia.py b/examples/services/cartesia.py similarity index 100% rename from examples/services/speech/cartesia.py rename to examples/services/cartesia.py diff --git a/examples/services/speech/deepgram-flux-sagemaker.py b/examples/services/deepgram-flux-sagemaker.py similarity index 100% rename from examples/services/speech/deepgram-flux-sagemaker.py rename to examples/services/deepgram-flux-sagemaker.py diff --git a/examples/services/speech/deepgram-flux.py b/examples/services/deepgram-flux.py similarity index 100% rename from examples/services/speech/deepgram-flux.py rename to examples/services/deepgram-flux.py diff --git a/examples/services/speech/deepgram-http.py b/examples/services/deepgram-http.py similarity index 100% rename from examples/services/speech/deepgram-http.py rename to examples/services/deepgram-http.py diff --git a/examples/services/speech/deepgram-sagemaker.py b/examples/services/deepgram-sagemaker.py similarity index 100% rename from examples/services/speech/deepgram-sagemaker.py rename to examples/services/deepgram-sagemaker.py diff --git a/examples/services/speech/deepgram-vad.py b/examples/services/deepgram-vad.py similarity index 100% rename from examples/services/speech/deepgram-vad.py rename to examples/services/deepgram-vad.py diff --git a/examples/services/speech/deepgram.py b/examples/services/deepgram.py similarity index 100% rename from examples/services/speech/deepgram.py rename to examples/services/deepgram.py diff --git a/examples/services/speech/elevenlabs-http.py b/examples/services/elevenlabs-http.py similarity index 100% rename from examples/services/speech/elevenlabs-http.py rename to examples/services/elevenlabs-http.py diff --git a/examples/services/speech/elevenlabs.py b/examples/services/elevenlabs.py similarity index 100% rename from examples/services/speech/elevenlabs.py rename to examples/services/elevenlabs.py diff --git a/examples/services/speech/fal.py b/examples/services/fal.py similarity index 100% rename from examples/services/speech/fal.py rename to examples/services/fal.py diff --git a/examples/services/speech/fish.py b/examples/services/fish.py similarity index 100% rename from examples/services/speech/fish.py rename to examples/services/fish.py diff --git a/examples/services/speech/gladia-vad.py b/examples/services/gladia-vad.py similarity index 100% rename from examples/services/speech/gladia-vad.py rename to examples/services/gladia-vad.py diff --git a/examples/services/speech/gladia.py b/examples/services/gladia.py similarity index 100% rename from examples/services/speech/gladia.py rename to examples/services/gladia.py diff --git a/examples/services/speech/google-audio-in.py b/examples/services/google-audio-in.py similarity index 100% rename from examples/services/speech/google-audio-in.py rename to examples/services/google-audio-in.py diff --git a/examples/services/speech/google-gemini-tts.py b/examples/services/google-gemini-tts.py similarity index 100% rename from examples/services/speech/google-gemini-tts.py rename to examples/services/google-gemini-tts.py diff --git a/examples/services/speech/google-http.py b/examples/services/google-http.py similarity index 100% rename from examples/services/speech/google-http.py rename to examples/services/google-http.py diff --git a/examples/services/speech/google-image.py b/examples/services/google-image.py similarity index 100% rename from examples/services/speech/google-image.py rename to examples/services/google-image.py diff --git a/examples/services/speech/google.py b/examples/services/google.py similarity index 100% rename from examples/services/speech/google.py rename to examples/services/google.py diff --git a/examples/services/speech/gradium.py b/examples/services/gradium.py similarity index 100% rename from examples/services/speech/gradium.py rename to examples/services/gradium.py diff --git a/examples/services/speech/groq.py b/examples/services/groq.py similarity index 100% rename from examples/services/speech/groq.py rename to examples/services/groq.py diff --git a/examples/services/speech/hume.py b/examples/services/hume.py similarity index 100% rename from examples/services/speech/hume.py rename to examples/services/hume.py diff --git a/examples/services/speech/inworld-http.py b/examples/services/inworld-http.py similarity index 100% rename from examples/services/speech/inworld-http.py rename to examples/services/inworld-http.py diff --git a/examples/services/speech/inworld.py b/examples/services/inworld.py similarity index 100% rename from examples/services/speech/inworld.py rename to examples/services/inworld.py diff --git a/examples/services/speech/kokoro.py b/examples/services/kokoro.py similarity index 100% rename from examples/services/speech/kokoro.py rename to examples/services/kokoro.py diff --git a/examples/services/speech/krisp-viva.py b/examples/services/krisp-viva.py similarity index 100% rename from examples/services/speech/krisp-viva.py rename to examples/services/krisp-viva.py diff --git a/examples/services/speech/langchain.py b/examples/services/langchain.py similarity index 100% rename from examples/services/speech/langchain.py rename to examples/services/langchain.py diff --git a/examples/services/speech/lmnt.py b/examples/services/lmnt.py similarity index 100% rename from examples/services/speech/lmnt.py rename to examples/services/lmnt.py diff --git a/examples/services/speech/minimax.py b/examples/services/minimax.py similarity index 100% rename from examples/services/speech/minimax.py rename to examples/services/minimax.py diff --git a/examples/services/speech/neuphonic-http.py b/examples/services/neuphonic-http.py similarity index 100% rename from examples/services/speech/neuphonic-http.py rename to examples/services/neuphonic-http.py diff --git a/examples/services/speech/neuphonic.py b/examples/services/neuphonic.py similarity index 100% rename from examples/services/speech/neuphonic.py rename to examples/services/neuphonic.py diff --git a/examples/services/speech/nvidia.py b/examples/services/nvidia.py similarity index 100% rename from examples/services/speech/nvidia.py rename to examples/services/nvidia.py diff --git a/examples/services/speech/openai-http.py b/examples/services/openai-http.py similarity index 100% rename from examples/services/speech/openai-http.py rename to examples/services/openai-http.py diff --git a/examples/services/speech/openai-responses-http.py b/examples/services/openai-responses-http.py similarity index 100% rename from examples/services/speech/openai-responses-http.py rename to examples/services/openai-responses-http.py diff --git a/examples/services/speech/openai-responses.py b/examples/services/openai-responses.py similarity index 100% rename from examples/services/speech/openai-responses.py rename to examples/services/openai-responses.py diff --git a/examples/services/speech/openai.py b/examples/services/openai.py similarity index 100% rename from examples/services/speech/openai.py rename to examples/services/openai.py diff --git a/examples/services/speech/piper.py b/examples/services/piper.py similarity index 100% rename from examples/services/speech/piper.py rename to examples/services/piper.py diff --git a/examples/services/speech/resemble.py b/examples/services/resemble.py similarity index 100% rename from examples/services/speech/resemble.py rename to examples/services/resemble.py diff --git a/examples/services/speech/rime-http.py b/examples/services/rime-http.py similarity index 100% rename from examples/services/speech/rime-http.py rename to examples/services/rime-http.py diff --git a/examples/services/speech/rime.py b/examples/services/rime.py similarity index 100% rename from examples/services/speech/rime.py rename to examples/services/rime.py diff --git a/examples/services/speech/sarvam-http.py b/examples/services/sarvam-http.py similarity index 100% rename from examples/services/speech/sarvam-http.py rename to examples/services/sarvam-http.py diff --git a/examples/services/speech/sarvam.py b/examples/services/sarvam.py similarity index 100% rename from examples/services/speech/sarvam.py rename to examples/services/sarvam.py diff --git a/examples/services/speech/smallest.py b/examples/services/smallest.py similarity index 100% rename from examples/services/speech/smallest.py rename to examples/services/smallest.py diff --git a/examples/services/speech/soniox.py b/examples/services/soniox.py similarity index 100% rename from examples/services/speech/soniox.py rename to examples/services/soniox.py diff --git a/examples/services/speech/speechmatics-vad.py b/examples/services/speechmatics-vad.py similarity index 100% rename from examples/services/speech/speechmatics-vad.py rename to examples/services/speechmatics-vad.py diff --git a/examples/services/speech/speechmatics.py b/examples/services/speechmatics.py similarity index 100% rename from examples/services/speech/speechmatics.py rename to examples/services/speechmatics.py diff --git a/examples/services/speech/xai.py b/examples/services/xai.py similarity index 100% rename from examples/services/speech/xai.py rename to examples/services/xai.py diff --git a/examples/services/speech/xtts.py b/examples/services/xtts.py similarity index 100% rename from examples/services/speech/xtts.py rename to examples/services/xtts.py diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 3c540ff87..38cde2b11 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -97,60 +97,60 @@ EVAL_COMPLETE_TURN = EvalConfig( TESTS_SPEECH = [ - ("services/speech/cartesia.py", EVAL_SIMPLE_MATH), - ("services/speech/cartesia-http.py", EVAL_SIMPLE_MATH), - ("services/speech/speechmatics.py", EVAL_SIMPLE_MATH), - ("services/speech/speechmatics-vad.py", EVAL_SIMPLE_MATH), - ("services/speech/langchain.py", EVAL_SIMPLE_MATH), - ("services/speech/deepgram.py", EVAL_SIMPLE_MATH), - ("services/speech/deepgram-flux.py", EVAL_SIMPLE_MATH), - ("services/speech/deepgram-http.py", EVAL_SIMPLE_MATH), - ("services/speech/elevenlabs.py", EVAL_SIMPLE_MATH), - ("services/speech/elevenlabs-http.py", EVAL_SIMPLE_MATH), - ("services/speech/xai.py", EVAL_SIMPLE_MATH), - ("services/speech/azure.py", EVAL_SIMPLE_MATH), - ("services/speech/azure-http.py", EVAL_SIMPLE_MATH), - ("services/speech/openai.py", EVAL_SIMPLE_MATH), - ("services/speech/openai-http.py", EVAL_SIMPLE_MATH), - ("services/speech/gladia.py", EVAL_SIMPLE_MATH), - ("services/speech/gladia-vad.py", EVAL_SIMPLE_MATH), - ("services/speech/lmnt.py", EVAL_SIMPLE_MATH), - ("services/speech/groq.py", EVAL_SIMPLE_MATH), - ("services/speech/aws.py", EVAL_SIMPLE_MATH), - ("services/speech/aws-strands.py", EVAL_WEATHER), - ("services/speech/google-gemini-tts.py", EVAL_SIMPLE_MATH), - ("services/speech/google.py", EVAL_SIMPLE_MATH), - ("services/speech/google-http.py", EVAL_SIMPLE_MATH), - ("services/speech/assemblyai.py", EVAL_SIMPLE_MATH), - ("services/speech/krisp-viva.py", EVAL_SIMPLE_MATH), - ("services/speech/rime.py", EVAL_SIMPLE_MATH), - ("services/speech/rime-http.py", EVAL_SIMPLE_MATH), - ("services/speech/nvidia.py", EVAL_SIMPLE_MATH), - ("services/speech/google-audio-in.py", EVAL_SIMPLE_MATH), - ("services/speech/fish.py", EVAL_SIMPLE_MATH), - ("services/speech/neuphonic.py", EVAL_SIMPLE_MATH), - ("services/speech/neuphonic-http.py", EVAL_SIMPLE_MATH), - ("services/speech/fal.py", EVAL_SIMPLE_MATH), - ("services/speech/minimax.py", EVAL_SIMPLE_MATH), - ("services/speech/sarvam.py", EVAL_SIMPLE_MATH), - ("services/speech/sarvam-http.py", EVAL_SIMPLE_MATH), - ("services/speech/soniox.py", EVAL_SIMPLE_MATH), - ("services/speech/inworld.py", EVAL_SIMPLE_MATH), - ("services/speech/inworld-http.py", EVAL_SIMPLE_MATH), - ("services/speech/asyncai.py", EVAL_SIMPLE_MATH), - ("services/speech/asyncai-http.py", EVAL_SIMPLE_MATH), - ("services/speech/aicoustics.py", EVAL_SIMPLE_MATH), - ("services/speech/hume.py", EVAL_SIMPLE_MATH), - ("services/speech/gradium.py", EVAL_SIMPLE_MATH), - ("services/speech/camb.py", EVAL_SIMPLE_MATH), - ("services/speech/piper.py", EVAL_SIMPLE_MATH), - ("services/speech/kokoro.py", EVAL_SIMPLE_MATH), - ("services/speech/resemble.py", EVAL_SIMPLE_MATH), - ("services/speech/smallest.py", EVAL_SIMPLE_MATH), - ("services/speech/openai-responses.py", EVAL_SIMPLE_MATH), - ("services/speech/openai-responses-http.py", EVAL_SIMPLE_MATH), + ("services/cartesia.py", EVAL_SIMPLE_MATH), + ("services/cartesia-http.py", EVAL_SIMPLE_MATH), + ("services/speechmatics.py", EVAL_SIMPLE_MATH), + ("services/speechmatics-vad.py", EVAL_SIMPLE_MATH), + ("services/langchain.py", EVAL_SIMPLE_MATH), + ("services/deepgram.py", EVAL_SIMPLE_MATH), + ("services/deepgram-flux.py", EVAL_SIMPLE_MATH), + ("services/deepgram-http.py", EVAL_SIMPLE_MATH), + ("services/elevenlabs.py", EVAL_SIMPLE_MATH), + ("services/elevenlabs-http.py", EVAL_SIMPLE_MATH), + ("services/xai.py", EVAL_SIMPLE_MATH), + ("services/azure.py", EVAL_SIMPLE_MATH), + ("services/azure-http.py", EVAL_SIMPLE_MATH), + ("services/openai.py", EVAL_SIMPLE_MATH), + ("services/openai-http.py", EVAL_SIMPLE_MATH), + ("services/gladia.py", EVAL_SIMPLE_MATH), + ("services/gladia-vad.py", EVAL_SIMPLE_MATH), + ("services/lmnt.py", EVAL_SIMPLE_MATH), + ("services/groq.py", EVAL_SIMPLE_MATH), + ("services/aws.py", EVAL_SIMPLE_MATH), + ("services/aws-strands.py", EVAL_WEATHER), + ("services/google-gemini-tts.py", EVAL_SIMPLE_MATH), + ("services/google.py", EVAL_SIMPLE_MATH), + ("services/google-http.py", EVAL_SIMPLE_MATH), + ("services/assemblyai.py", EVAL_SIMPLE_MATH), + ("services/krisp-viva.py", EVAL_SIMPLE_MATH), + ("services/rime.py", EVAL_SIMPLE_MATH), + ("services/rime-http.py", EVAL_SIMPLE_MATH), + ("services/nvidia.py", EVAL_SIMPLE_MATH), + ("services/google-audio-in.py", EVAL_SIMPLE_MATH), + ("services/fish.py", EVAL_SIMPLE_MATH), + ("services/neuphonic.py", EVAL_SIMPLE_MATH), + ("services/neuphonic-http.py", EVAL_SIMPLE_MATH), + ("services/fal.py", EVAL_SIMPLE_MATH), + ("services/minimax.py", EVAL_SIMPLE_MATH), + ("services/sarvam.py", EVAL_SIMPLE_MATH), + ("services/sarvam-http.py", EVAL_SIMPLE_MATH), + ("services/soniox.py", EVAL_SIMPLE_MATH), + ("services/inworld.py", EVAL_SIMPLE_MATH), + ("services/inworld-http.py", EVAL_SIMPLE_MATH), + ("services/asyncai.py", EVAL_SIMPLE_MATH), + ("services/asyncai-http.py", EVAL_SIMPLE_MATH), + ("services/aicoustics.py", EVAL_SIMPLE_MATH), + ("services/hume.py", EVAL_SIMPLE_MATH), + ("services/gradium.py", EVAL_SIMPLE_MATH), + ("services/camb.py", EVAL_SIMPLE_MATH), + ("services/piper.py", EVAL_SIMPLE_MATH), + ("services/kokoro.py", EVAL_SIMPLE_MATH), + ("services/resemble.py", EVAL_SIMPLE_MATH), + ("services/smallest.py", EVAL_SIMPLE_MATH), + ("services/openai-responses.py", EVAL_SIMPLE_MATH), + ("services/openai-responses-http.py", EVAL_SIMPLE_MATH), # Needs a local XTTS docker instance running. - # ("services/speech/xtts.py", EVAL_SIMPLE_MATH), + # ("services/xtts.py", EVAL_SIMPLE_MATH), ] TESTS_VISION = [ @@ -169,44 +169,44 @@ TESTS_VISION = [ TESTS_FUNCTION_CALLING = [ ("getting-started/07-function-calling.py", EVAL_WEATHER), ("getting-started/07-function-calling.py", EVAL_WEATHER_AND_RESTAURANT), - ("services/function-calling/openai-responses.py", EVAL_WEATHER), - ("services/function-calling/openai-responses.py", EVAL_WEATHER_AND_RESTAURANT), - ("services/function-calling/openai-responses-http.py", EVAL_WEATHER), - ("services/function-calling/openai-responses-http.py", EVAL_WEATHER_AND_RESTAURANT), - ("services/function-calling/anthropic.py", EVAL_WEATHER), - ("services/function-calling/anthropic.py", EVAL_WEATHER_AND_RESTAURANT), - ("services/function-calling/openai.py", EVAL_WEATHER), - ("services/function-calling/google.py", EVAL_WEATHER), - ("services/function-calling/google.py", EVAL_WEATHER_AND_RESTAURANT), - ("services/function-calling/groq.py", EVAL_WEATHER), - ("services/function-calling/grok.py", EVAL_WEATHER), - ("services/function-calling/azure.py", EVAL_WEATHER), - ("services/function-calling/fireworks.py", EVAL_WEATHER), - ("services/function-calling/nvidia.py", EVAL_WEATHER), - ("services/function-calling/cerebras.py", EVAL_WEATHER), - ("services/function-calling/openrouter.py", EVAL_WEATHER), - ("services/function-calling/perplexity.py", EVAL_WEATHER), - ("services/function-calling/google-vertex.py", EVAL_WEATHER), - ("services/function-calling/qwen.py", EVAL_WEATHER), - ("services/function-calling/aws.py", EVAL_WEATHER), - ("services/function-calling/sambanova.py", EVAL_WEATHER), - ("services/function-calling/aws.py", EVAL_WEATHER_AND_RESTAURANT), - ("services/function-calling/nebius.py", EVAL_WEATHER), - ("services/function-calling/mistral.py", EVAL_WEATHER), - ("services/function-calling/sarvam.py", EVAL_WEATHER), - ("services/function-calling/novita.py", EVAL_WEATHER), + ("function-calling/openai-responses.py", EVAL_WEATHER), + ("function-calling/openai-responses.py", EVAL_WEATHER_AND_RESTAURANT), + ("function-calling/openai-responses-http.py", EVAL_WEATHER), + ("function-calling/openai-responses-http.py", EVAL_WEATHER_AND_RESTAURANT), + ("function-calling/anthropic.py", EVAL_WEATHER), + ("function-calling/anthropic.py", EVAL_WEATHER_AND_RESTAURANT), + ("function-calling/openai.py", EVAL_WEATHER), + ("function-calling/google.py", EVAL_WEATHER), + ("function-calling/google.py", EVAL_WEATHER_AND_RESTAURANT), + ("function-calling/groq.py", EVAL_WEATHER), + ("function-calling/grok.py", EVAL_WEATHER), + ("function-calling/azure.py", EVAL_WEATHER), + ("function-calling/fireworks.py", EVAL_WEATHER), + ("function-calling/nvidia.py", EVAL_WEATHER), + ("function-calling/cerebras.py", EVAL_WEATHER), + ("function-calling/openrouter.py", EVAL_WEATHER), + ("function-calling/perplexity.py", EVAL_WEATHER), + ("function-calling/google-vertex.py", EVAL_WEATHER), + ("function-calling/qwen.py", EVAL_WEATHER), + ("function-calling/aws.py", EVAL_WEATHER), + ("function-calling/sambanova.py", EVAL_WEATHER), + ("function-calling/aws.py", EVAL_WEATHER_AND_RESTAURANT), + ("function-calling/nebius.py", EVAL_WEATHER), + ("function-calling/mistral.py", EVAL_WEATHER), + ("function-calling/sarvam.py", EVAL_WEATHER), + ("function-calling/novita.py", EVAL_WEATHER), # Video - ("services/function-calling/anthropic-video.py", EVAL_VISION_CAMERA), - ("services/function-calling/aws-video.py", EVAL_VISION_CAMERA), - ("services/function-calling/google-video.py", EVAL_VISION_CAMERA), - ("services/function-calling/moondream-video.py", EVAL_VISION_CAMERA), - ("services/function-calling/openai-video.py", EVAL_VISION_CAMERA), - ("services/function-calling/openai-responses-video.py", EVAL_VISION_CAMERA), - ("services/function-calling/openai-responses-video-http.py", EVAL_VISION_CAMERA), + ("function-calling/anthropic-video.py", EVAL_VISION_CAMERA), + ("function-calling/aws-video.py", EVAL_VISION_CAMERA), + ("function-calling/google-video.py", EVAL_VISION_CAMERA), + ("function-calling/moondream-video.py", EVAL_VISION_CAMERA), + ("function-calling/openai-video.py", EVAL_VISION_CAMERA), + ("function-calling/openai-responses-video.py", EVAL_VISION_CAMERA), + ("function-calling/openai-responses-video-http.py", EVAL_VISION_CAMERA), # Currently not working. - # ("services/function-calling/together.py", EVAL_WEATHER), - # ("services/function-calling/deepseek.py", EVAL_WEATHER), - # ("services/function-calling/gemini-openai-format.py", EVAL_WEATHER), + # ("function-calling/together.py", EVAL_WEATHER), + # ("function-calling/deepseek.py", EVAL_WEATHER), + # ("function-calling/gemini-openai-format.py", EVAL_WEATHER), ] TESTS_FEATURES = [