diff --git a/README.md b/README.md index 881b0ed5e..8af4f942c 100644 --- a/README.md +++ b/README.md @@ -81,19 +81,19 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout ## 🧩 Available services -| Category | Services | -| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [Hathora](https://docs.pipecat.ai/server/services/stt/hathora), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | -| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | -| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hathora](https://docs.pipecat.ai/server/services/tts/hathora), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [Resemble](https://docs.pipecat.ai/server/services/tts/resemble), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | -| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | -| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) | -| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [LemonSlice](https://docs.pipecat.ai/server/services/video/lemonslice), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | -| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | -| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/google-imagen), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | -| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | +| Category | Services | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | +| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [Resemble](https://docs.pipecat.ai/server/services/tts/resemble), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | +| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | +| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) | +| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [LemonSlice](https://docs.pipecat.ai/server/services/video/lemonslice), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | +| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | +| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/google-imagen), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | +| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | 📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) diff --git a/env.example b/env.example index 81f3f895d..dda5bb084 100644 --- a/env.example +++ b/env.example @@ -86,9 +86,6 @@ GROK_API_KEY=... # Groq GROQ_API_KEY=... -# Hathora -HATHORA_API_KEY=... - # Heygen HEYGEN_API_KEY=... HEYGEN_LIVE_AVATAR_API_KEY=... diff --git a/examples/foundational/07zh-interruptible-hathora.py b/examples/foundational/07zh-interruptible-hathora.py deleted file mode 100644 index 08749bf1a..000000000 --- a/examples/foundational/07zh-interruptible-hathora.py +++ /dev/null @@ -1,123 +0,0 @@ -# -# Copyright (c) 2024–2026, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.hathora.stt import HathoraSTTService -from pipecat.services.hathora.tts import HathoraTTSService -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams - -load_dotenv(override=True) - -# We use lambdas to defer transport parameter creation until the transport -# type is selected at runtime. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = HathoraSTTService( - model="nvidia-parakeet-tdt-0.6b-v3", - ) - - tts = HathoraTTSService( - model="hexgrad-kokoro-82m", - ) - - # See https://models.hathora.dev/model/qwen3-30b-a3b - llm = OpenAILLMService( - base_url="https://app-362f7ca1-6975-4e18-a605-ab202bf2c315.app.hathora.dev/v1", - api_key=os.getenv("HATHORA_API_KEY"), - model=None, - system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", - ) - - context = LLMContext() - context_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), - ) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - # Kick off the conversation. - context.add_message({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/55zg-update-settings-hathora-tts.py b/examples/foundational/55zg-update-settings-hathora-tts.py deleted file mode 100644 index 2294809e2..000000000 --- a/examples/foundational/55zg-update-settings-hathora-tts.py +++ /dev/null @@ -1,121 +0,0 @@ -# -# Copyright (c) 2024-2026, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.hathora.tts import HathoraTTSService, HathoraTTSSettings -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams - -load_dotenv(override=True) - -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = HathoraTTSService( - api_key=os.getenv("HATHORA_API_KEY"), - model="hexgrad-kokoro-82m", - ) - - llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), - system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", - ) - - context = LLMContext() - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), - ) - - pipeline = Pipeline( - [ - transport.input(), - stt, - user_aggregator, - llm, - tts, - transport.output(), - assistant_aggregator, - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - context.add_message({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([LLMRunFrame()]) - - await asyncio.sleep(10) - logger.info("Updating Hathora TTS settings: speed=1.5") - await task.queue_frame(TTSUpdateSettingsFrame(delta=HathoraTTSSettings(speed=1.5))) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/55zs-update-settings-hathora-stt.py b/examples/foundational/55zs-update-settings-hathora-stt.py deleted file mode 100644 index 818772b75..000000000 --- a/examples/foundational/55zs-update-settings-hathora-stt.py +++ /dev/null @@ -1,127 +0,0 @@ -# -# Copyright (c) 2024-2026, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.hathora.stt import HathoraSTTService, HathoraSTTSettings -from pipecat.services.hathora.utils import ConfigOption -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transcriptions.language import Language -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams - -load_dotenv(override=True) - -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = HathoraSTTService( - api_key=os.getenv("HATHORA_API_KEY"), model="nvidia-parakeet-tdt-0.6b-v3" - ) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), - system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", - ) - - context = LLMContext() - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), - ) - - pipeline = Pipeline( - [ - transport.input(), - stt, - user_aggregator, - llm, - tts, - transport.output(), - assistant_aggregator, - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - context.add_message({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([LLMRunFrame()]) - - await asyncio.sleep(10) - logger.info("Updating Hathora STT settings: language=es") - await task.queue_frame( - STTUpdateSettingsFrame(delta=HathoraSTTSettings(language=Language.ES)) - ) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 77fc23a33..e473403b8 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -144,7 +144,6 @@ TESTS_07 = [ ("07ze-interruptible-hume.py", EVAL_SIMPLE_MATH), ("07zf-interruptible-gradium.py", EVAL_SIMPLE_MATH), ("07zg-interruptible-camb.py", EVAL_SIMPLE_MATH), - ("07zh-interruptible-hathora.py", EVAL_SIMPLE_MATH), ("07zi-interruptible-piper.py", EVAL_SIMPLE_MATH), ("07zj-interruptible-kokoro.py", EVAL_SIMPLE_MATH), # Needs a local XTTS docker instance running. diff --git a/src/pipecat/services/hathora/__init__.py b/src/pipecat/services/hathora/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/pipecat/services/hathora/stt.py b/src/pipecat/services/hathora/stt.py deleted file mode 100644 index 27f1aebfb..000000000 --- a/src/pipecat/services/hathora/stt.py +++ /dev/null @@ -1,176 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -"""[Hathora-hosted](https://models.hathora.dev) speech-to-text services.""" - -import base64 -import os -from dataclasses import dataclass, field -from typing import AsyncGenerator, Optional - -import aiohttp -from pydantic import BaseModel - -from pipecat.frames.frames import ( - ErrorFrame, - Frame, - TranscriptionFrame, -) -from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven -from pipecat.services.stt_latency import HATHORA_TTFS_P99 -from pipecat.services.stt_service import SegmentedSTTService -from pipecat.transcriptions.language import Language -from pipecat.utils.time import time_now_iso8601 -from pipecat.utils.tracing.service_decorators import traced_stt - -from .utils import ConfigOption - - -@dataclass -class HathoraSTTSettings(STTSettings): - """Settings for the Hathora STT service. - - Parameters: - config: Some models support additional config, refer to - `docs `_ for each model to see - what is supported. - """ - - config: list[ConfigOption] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) - - -class HathoraSTTService(SegmentedSTTService): - """This service supports several different speech-to-text models hosted by Hathora. - - [Documentation](https://models.hathora.dev) - """ - - _settings: HathoraSTTSettings - - class InputParams(BaseModel): - """Optional input parameters for Hathora STT configuration. - - Parameters: - language: Language code (if supported by model). - config: Some models support additional config, refer to - [docs](https://models.hathora.dev) for each model to see - what is supported. - """ - - language: Optional[str] = None - config: Optional[list[ConfigOption]] = None - - def __init__( - self, - *, - model: str, - sample_rate: Optional[int] = None, - api_key: Optional[str] = None, - base_url: str = "https://api.models.hathora.dev/inference/v1/stt", - params: Optional[InputParams] = None, - ttfs_p99_latency: Optional[float] = HATHORA_TTFS_P99, - **kwargs, - ): - """Initialize the Hathora STT service. - - Args: - model: Model to use; find available models - [here](https://models.hathora.dev). - sample_rate: The sample rate for audio input. If None, will be determined - from the start frame. - api_key: API key for authentication with the Hathora service; - provision one [here](https://models.hathora.dev/tokens). - base_url: Base API URL for the Hathora STT service. - params: Configuration parameters. - ttfs_p99_latency: P99 latency from speech end to final transcript in seconds. - Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark - **kwargs: Additional arguments passed to the parent class. - """ - params = params or HathoraSTTService.InputParams() - - super().__init__( - sample_rate=sample_rate, - ttfs_p99_latency=ttfs_p99_latency, - settings=HathoraSTTSettings( - model=model, - language=params.language, - config=params.config, - ), - **kwargs, - ) - self._api_key = api_key or os.getenv("HATHORA_API_KEY") - self._base_url = base_url - - def can_generate_metrics(self) -> bool: - """Check if this service can generate processing metrics. - - Returns: - True - """ - return True - - @traced_stt - async def _handle_transcription( - self, transcript: str, is_final: bool, language: Optional[Language] = None - ): - """Handle a transcription result with tracing.""" - pass - - async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Run speech-to-text on the provided audio data. - - Args: - audio: Raw audio bytes to transcribe. - - Yields: - Frame: Frames containing transcription results (typically TextFrame). - """ - try: - await self.start_processing_metrics() - - url = f"{self._base_url}" - - payload = { - "model": self._settings.model, - } - - if self._settings.language is not None: - payload["language"] = self._settings.language - if self._settings.config is not None: - payload["model_config"] = [ - {"name": option.name, "value": option.value} for option in self._settings.config - ] - - base64_audio = base64.b64encode(audio).decode("utf-8") - payload["audio"] = base64_audio - - async with aiohttp.ClientSession() as session: - async with session.post( - url, - headers={"Authorization": f"Bearer {self._api_key}"}, - json=payload, - ) as resp: - response = await resp.json() - - if response and "text" in response: - text = response["text"].strip() - if text: # Only yield non-empty text - # Hathora's API currently doesn't return language info - # so we default to the requested language or "en" - response_language = self._settings.language or "en" - await self._handle_transcription(text, True, response_language) - yield TranscriptionFrame( - text, - self._user_id, - time_now_iso8601(), - Language(response_language), - result=response, - ) - - await self.stop_processing_metrics() - - except Exception as e: - yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/src/pipecat/services/hathora/tts.py b/src/pipecat/services/hathora/tts.py deleted file mode 100644 index 3fb9e747b..000000000 --- a/src/pipecat/services/hathora/tts.py +++ /dev/null @@ -1,191 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -"""[Hathora-hosted](https://models.hathora.dev) text-to-speech services.""" - -import io -import os -import wave -from dataclasses import dataclass, field -from typing import AsyncGenerator, Optional, Tuple - -import aiohttp -from pydantic import BaseModel - -from pipecat.frames.frames import ( - ErrorFrame, - Frame, - TTSAudioRawFrame, - TTSStartedFrame, - TTSStoppedFrame, -) -from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven -from pipecat.services.tts_service import TTSService -from pipecat.utils.tracing.service_decorators import traced_tts - -from .utils import ConfigOption - - -def _decode_audio_payload( - audio_bytes: bytes, - *, - fallback_sample_rate: int = 24000, - fallback_channels: int = 1, -) -> Tuple[bytes, int, int]: - """Convert a WAV/PCM payload into raw PCM samples for TTSAudioRawFrame.""" - try: - with wave.open(io.BytesIO(audio_bytes), "rb") as wav_reader: - channels = wav_reader.getnchannels() - sample_rate = wav_reader.getframerate() - frames = wav_reader.readframes(wav_reader.getnframes()) - return frames, sample_rate, channels - except (wave.Error, EOFError): - # If the payload is already raw PCM, just pass it through. - return audio_bytes, fallback_sample_rate, fallback_channels - - -@dataclass -class HathoraTTSSettings(TTSSettings): - """Settings for Hathora TTS service. - - Parameters: - speed: Speech speed multiplier (if supported by model). - config: Some models support additional config, refer to - [docs](https://models.hathora.dev) for each model to see - what is supported. - """ - - speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) - config: list[ConfigOption] | _NotGiven = field(default_factory=lambda: NOT_GIVEN) - - -class HathoraTTSService(TTSService): - """This service supports several different text-to-speech models hosted by Hathora. - - [Documentation](https://models.hathora.dev) - """ - - _settings: HathoraTTSSettings - - class InputParams(BaseModel): - """Optional input parameters for Hathora TTS configuration. - - Parameters: - speed: Speech speed multiplier (if supported by model). - config: Some models support additional config, refer to - [docs](https://models.hathora.dev) for each model to see - what is supported. - """ - - speed: Optional[float] = None - config: Optional[list[ConfigOption]] = None - - def __init__( - self, - *, - model: str, - voice_id: Optional[str] = None, - sample_rate: Optional[int] = None, - api_key: Optional[str] = None, - base_url: str = "https://api.models.hathora.dev/inference/v1/tts", - params: Optional[InputParams] = None, - **kwargs, - ): - """Initialize the Hathora TTS service. - - Args: - model: Model to use; find available models - [here](https://models.hathora.dev). - voice_id: Voice to use for synthesis (if supported by model). - sample_rate: Output sample rate for generated audio. - api_key: API key for authentication with the Hathora service; - provision one [here](https://models.hathora.dev/tokens). - base_url: Base API URL for the Hathora TTS service. - params: Configuration parameters. - **kwargs: Additional arguments passed to the parent class. - """ - params = params or HathoraTTSService.InputParams() - - super().__init__( - sample_rate=sample_rate, - settings=HathoraTTSSettings( - model=model, - voice=voice_id, - language=None, # Not applicable here - speed=params.speed, - config=params.config, - ), - **kwargs, - ) - self._api_key = api_key or os.getenv("HATHORA_API_KEY") - self._base_url = base_url - - def can_generate_metrics(self) -> bool: - """Check if this service can generate processing metrics. - - Returns: - True - """ - return True - - @traced_tts - async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: - """Run text-to-speech synthesis on the provided text. - - Args: - text: The text to synthesize into speech. - context_id: The context ID for tracking audio frames. - - Yields: - Frame: Audio frames containing the synthesized speech. - """ - try: - await self.start_processing_metrics() - await self.start_ttfb_metrics() - - url = f"{self._base_url}" - - payload = {"model": self._settings.model, "text": text} - - if self._settings.voice is not None: - payload["voice"] = self._settings.voice - if self._settings.speed is not None: - payload["speed"] = self._settings.speed - if self._settings.config is not None: - payload["model_config"] = [ - {"name": option.name, "value": option.value} for option in self._settings.config - ] - - yield TTSStartedFrame(context_id=context_id) - - async with aiohttp.ClientSession() as session: - async with session.post( - url, - headers={"Authorization": f"Bearer {self._api_key}"}, - json=payload, - ) as resp: - audio_data = await resp.read() - - pcm_audio, sample_rate, num_channels = _decode_audio_payload( - audio_data, - fallback_sample_rate=self.sample_rate, - ) - - frame = TTSAudioRawFrame( - audio=pcm_audio, - sample_rate=self.sample_rate, - num_channels=num_channels, - context_id=context_id, - ) - - yield frame - - except Exception as e: - yield ErrorFrame(error=f"Unknown error occurred: {e}") - finally: - await self.stop_ttfb_metrics() - await self.stop_processing_metrics() - yield TTSStoppedFrame(context_id=context_id) diff --git a/src/pipecat/services/hathora/utils.py b/src/pipecat/services/hathora/utils.py deleted file mode 100644 index 6d8eb54b8..000000000 --- a/src/pipecat/services/hathora/utils.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -"""Utilities and types for [Hathora-hosted](https://models.hathora.dev) voice services.""" - -from dataclasses import dataclass - - -@dataclass -class ConfigOption: - """Extra configuration option passed into model_config for Hathora (if supported by model). - - Args: - name: Name of the configuration option. - value: Value of the configuration option. - """ - - name: str - value: str diff --git a/src/pipecat/services/stt_latency.py b/src/pipecat/services/stt_latency.py index bf895e1e4..351e041a6 100644 --- a/src/pipecat/services/stt_latency.py +++ b/src/pipecat/services/stt_latency.py @@ -40,7 +40,6 @@ GLADIA_TTFS_P99: float = 1.49 GOOGLE_TTFS_P99: float = 1.57 GRADIUM_TTFS_P99: float = 1.61 GROQ_TTFS_P99: float = 1.54 -HATHORA_TTFS_P99: float = 0.87 OPENAI_TTFS_P99: float = 2.01 OPENAI_REALTIME_TTFS_P99: float = 1.66 SAMBANOVA_TTFS_P99: float = 2.20