From e5632a9339ae48a81977efe7e470d74cd2d07a02 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Wed, 17 Dec 2025 19:16:58 -0800 Subject: [PATCH] transition Hathora service to use the unified API and apply PR feedback add Hathora to root files Hathora run linter added hathora changelog --- README.md | 4 +- changelog/3169.added.md | 1 + env.example | 3 + ...thora.py => 07ag-interruptible-hathora.py} | 32 ++-- src/pipecat/services/hathora/__init__.py | 14 -- src/pipecat/services/hathora/stt.py | 108 ++++++----- src/pipecat/services/hathora/tts.py | 173 +++++------------- src/pipecat/services/hathora/utils.py | 22 +++ 8 files changed, 152 insertions(+), 205 deletions(-) create mode 100644 changelog/3169.added.md rename examples/foundational/{07af-interruptible-hathora.py => 07ag-interruptible-hathora.py} (80%) create mode 100644 src/pipecat/services/hathora/utils.py diff --git a/README.md b/README.md index 0fac27943..65a12634f 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,9 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout | Category | Services | | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [Hathora](https://docs.pipecat.ai/server/services/stt/hathora), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | -| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | +| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hathora](https://docs.pipecat.ai/server/services/tts/hathora), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | | Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | | Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) | diff --git a/changelog/3169.added.md b/changelog/3169.added.md new file mode 100644 index 000000000..d9e4bb9b9 --- /dev/null +++ b/changelog/3169.added.md @@ -0,0 +1 @@ +- Added Hathora service to support Hathora-hosted TTS and STT models (only non-streaming) \ No newline at end of file diff --git a/env.example b/env.example index 41badb571..c7b734ad6 100644 --- a/env.example +++ b/env.example @@ -85,6 +85,9 @@ GROK_API_KEY=... # Groq GROQ_API_KEY=... +# Hathora +HATHORA_API_KEY=... + # Heygen HEYGEN_API_KEY=... HEYGEN_LIVE_AVATAR_API_KEY=... diff --git a/examples/foundational/07af-interruptible-hathora.py b/examples/foundational/07ag-interruptible-hathora.py similarity index 80% rename from examples/foundational/07af-interruptible-hathora.py rename to examples/foundational/07ag-interruptible-hathora.py index 5a06b7bf0..bcde037d0 100644 --- a/examples/foundational/07af-interruptible-hathora.py +++ b/examples/foundational/07ag-interruptible-hathora.py @@ -21,8 +21,8 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.hathora.stt import ParakeetSTTService -from pipecat.services.hathora.tts import ChatterboxTTSService, KokoroTTSService +from pipecat.services.hathora.stt import HathoraSTTService +from pipecat.services.hathora.tts import HathoraTTSService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -38,13 +38,19 @@ transport_params = { audio_in_enabled=True, audio_out_enabled=True, vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), ), "webrtc": lambda: TransportParams( audio_in_enabled=True, audio_out_enabled=True, vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), ), } @@ -52,24 +58,14 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - # See https://models.hathora.dev/model/nvidia-parakeet-tdt-0.6b-v3 - stt = ParakeetTDTSTTService( - base_url="https://app-1c7bebb9-6977-4101-9619-833b251b86d1.app.hathora.dev/v1/transcribe", - api_key=os.getenv("HATHORA_API_KEY") + stt = HathoraSTTService( + model="nvidia-parakeet-tdt-0.6b-v3", ) - # See https://models.hathora.dev/model/hexgrad-kokoro-82m - tts = KokoroTTSService( - base_url="https://app-01312daf-6e53-4b9d-a4ad-13039f35adc4.app.hathora.dev/synthesize", - api_key=os.getenv("HATHORA_API_KEY"), + tts = HathoraTTSService( + model="hexgrad-kokoro-82m", ) - # See https://models.hathora.dev/model/resemble-ai-chatterbox - # tts = ChatterboxTTSService( - # base_url="https://app-efbc8fe2-df55-4f96-bbe3-74f6ea9d986b.app.hathora.dev/v1/generate", - # api_key=os.getenv("HATHORA_API_KEY") - # ) - # See https://models.hathora.dev/model/qwen3-30b-a3b llm = OpenAILLMService( base_url="https://app-362f7ca1-6975-4e18-a605-ab202bf2c315.app.hathora.dev/v1", diff --git a/src/pipecat/services/hathora/__init__.py b/src/pipecat/services/hathora/__init__.py index 11b7dbb55..e69de29bb 100644 --- a/src/pipecat/services/hathora/__init__.py +++ b/src/pipecat/services/hathora/__init__.py @@ -1,14 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import sys - -from pipecat.services import DeprecatedModuleProxy - -from .stt import * -from .tts import * - -sys.modules[__name__] = DeprecatedModuleProxy(globals(), "hathora", "hathora.[stt,tts]") diff --git a/src/pipecat/services/hathora/stt.py b/src/pipecat/services/hathora/stt.py index ef4b209af..010f38683 100644 --- a/src/pipecat/services/hathora/stt.py +++ b/src/pipecat/services/hathora/stt.py @@ -6,102 +6,126 @@ """[Hathora-hosted](https://models.hathora.dev) speech-to-text services.""" +import base64 import os -from typing import Optional +from typing import AsyncGenerator, Optional import aiohttp -from loguru import logger from pipecat.frames.frames import ( ErrorFrame, + Frame, TranscriptionFrame, ) from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_stt -class ParakeetTDTSTTService(SegmentedSTTService): - """Parakeet TDT is a multilingual automatic speech recognition model - with word-level timestamps. +from .utils import ConfigOption - This service uses the Hathora-hosted Parakeet model via the HTTP API. - [Documentation](https://models.hathora.dev/model/nvidia-parakeet-tdt-0.6b-v3) +class HathoraSTTService(SegmentedSTTService): + """This service supports several different speech-to-text models hosted by Hathora. + + [Documentation](https://models.hathora.dev) """ def __init__( self, *, - base_url = None, - api_key = None, - start_time: Optional[int] = None, - end_time: Optional[int] = None, + model: str, + language: Optional[str] = None, + model_config: Optional[list[ConfigOption]] = None, + base_url: str = "https://api.models.hathora.dev/inference/v1/stt", + api_key: Optional[str] = None, **kwargs, ): - """Initialize the Hathora-hosted Parakeet STT service. + """Initialize the Hathora STT service. Args: - base_url: Base URL for the Hathora Parakeet STT API. + model: Model to use; find available models + [here](https://models.hathora.dev). + language: Language code (if supported by model). + model_config: Some models support additional config, refer to + [docs](https://models.hathora.dev) for each model to see + what is supported. + base_url: Base API URL for the Hathora STT service. api_key: API key for authentication with the Hathora service; - provisiion one [here](https://models.hathora.dev/tokens). - start_time: Start time in seconds for the time window. - end_time: End time in seconds for the time window. + provision one [here](https://models.hathora.dev/tokens). + **kwargs: Additional arguments passed to the parent class. """ super().__init__( **kwargs, ) + self._model = model + self._language = language + self._model_config = model_config self._base_url = base_url - self._api_key = api_key - self._start_time = start_time - self._end_time = end_time + self._api_key = api_key or os.getenv("HATHORA_API_KEY") - def can_generate_metrics(self) -> bool: - return True + @traced_stt + async def _handle_transcription( + self, transcript: str, is_final: bool, language: Optional[Language] = None + ): + """Handle a transcription result with tracing.""" + pass - async def run_stt(self, audio: bytes): + @traced_stt + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Run speech-to-text on the provided audio data. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + Frame: Frames containing transcription results (typically TextFrame). + """ try: await self.start_processing_metrics() await self.start_ttfb_metrics() url = f"{self._base_url}" - url_query_params = [] - if self._start_time is not None: - url_query_params.append(f"start_time={self._start_time}") - if self._end_time is not None: - url_query_params.append(f"end_time={self._end_time}") - url_query_params.append(f"sample_rate={self.sample_rate}") + payload = { + "model": self._model, + } - if len(url_query_params) > 0: - url += "?" + "&".join(url_query_params) + if self._language is not None: + payload["language"] = self._language + if self._model_config is not None: + payload["model_config"] = [ + {"name": option.name, "value": option.value} for option in self._model_config + ] - api_key = self._api_key or os.getenv("HATHORA_API_KEY") - - form_data = aiohttp.FormData() - form_data.add_field("file", audio, filename="audio.wav", content_type="application/octet-stream") + base64_audio = base64.b64encode(audio).decode("utf-8") + payload["audio"] = base64_audio async with aiohttp.ClientSession() as session: async with session.post( url, - headers={"Authorization": f"Bearer {api_key}"}, - data=form_data, + headers={"Authorization": f"Bearer {self._api_key}"}, + json=payload, ) as resp: response = await resp.json() if response and "text" in response: text = response["text"].strip() if text: # Only yield non-empty text - await self.stop_ttfb_metrics() - await self.stop_processing_metrics() - logger.debug(f"Transcription: [{text}]") + # Hathora's API currently doesn't return language info + # so we default to the requested language or "en" + response_language = self._language or "en" + await self._handle_transcription(text, True, response_language) yield TranscriptionFrame( text, self._user_id, time_now_iso8601(), - Language("en"), # TODO: the parakeet hathora API doesn't accept a language but says it's multilingual + Language(response_language), result=response, ) + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() + except Exception as e: - logger.error(f"Hathora error: {e}") - yield ErrorFrame(f"Hathora error: {str(e)}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/src/pipecat/services/hathora/tts.py b/src/pipecat/services/hathora/tts.py index d9ec78793..85843923d 100644 --- a/src/pipecat/services/hathora/tts.py +++ b/src/pipecat/services/hathora/tts.py @@ -9,18 +9,22 @@ import io import os import wave -from typing import Optional, Tuple +from typing import AsyncGenerator, Optional, Tuple import aiohttp -from loguru import logger from pipecat.frames.frames import ( ErrorFrame, + Frame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, ) from pipecat.services.tts_service import TTSService +from pipecat.utils.tracing.service_decorators import traced_tts + +from .utils import ConfigOption + def _decode_audio_payload( audio_bytes: bytes, @@ -29,7 +33,6 @@ def _decode_audio_payload( fallback_channels: int = 1, ) -> Tuple[bytes, int, int]: """Convert a WAV/PCM payload into raw PCM samples for TTSAudioRawFrame.""" - try: with wave.open(io.BytesIO(audio_bytes), "rb") as wav_reader: channels = wav_reader.getnchannels() @@ -40,69 +43,82 @@ def _decode_audio_payload( # If the payload is already raw PCM, just pass it through. return audio_bytes, fallback_sample_rate, fallback_channels -class KokoroTTSService(TTSService): - """Kokoro is an open-weight TTS model with 82 million parameters. - This service uses the Hathora-hosted Kokoro model via the HTTP API. +class HathoraTTSService(TTSService): + """This service supports several different text-to-speech models hosted by Hathora. - [Documentation](https://models.hathora.dev/model/hexgrad-kokoro-82m) + [Documentation](https://models.hathora.dev) """ def __init__( self, *, - base_url = None, - api_key = None, + model: str, voice: Optional[str] = None, speed: Optional[float] = None, + model_config: Optional[list[ConfigOption]] = None, + base_url: str = "https://api.models.hathora.dev/inference/v1/tts", + api_key: Optional[str] = None, **kwargs, ): - """Initialize the Hathora-hosted Kokoro TTS service. + """Initialize the Hathora TTS service. Args: - base_url: Base URL for the Hathora Kokoro TTS API, . + model: Model to use; find available models + [here](https://models.hathora.dev). + voice: Voice to use for synthesis (if supported by model). + speed: Speech speed multiplier (if supported by model). + model_config: Some models support additional config, refer to + [docs](https://models.hathora.dev) for each model to see + what is supported. + base_url: Base API URL for the Hathora TTS service. api_key: API key for authentication with the Hathora service; - provisiion one [here](https://models.hathora.dev/tokens). - voice: Voice to use for synthesis (see the - [Hathora docs](https://models.hathora.dev/model/hexgrad-kokoro-82m) - for the default value; [list of voices](https://huggingface.co/hexgrad/Kokoro-82M/blob/main/VOICES.md)). - speed: Speech speed multiplier (0.5 = half speed, 2.0 = double speed, default: 1). + provision one [here](https://models.hathora.dev/tokens). + **kwargs: Additional arguments passed to the parent class. """ super().__init__( **kwargs, ) - self._base_url = base_url - self._api_key = api_key + self._model = model self._voice = voice self._speed = speed + self._model_config = model_config + self._base_url = base_url + self._api_key = api_key or os.getenv("HATHORA_API_KEY") - def can_generate_metrics(self) -> bool: - return True + @traced_tts + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Run text-to-speech synthesis on the provided text. - async def run_tts(self, text: str): + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ try: await self.start_processing_metrics() await self.start_ttfb_metrics() url = f"{self._base_url}" - api_key = self._api_key or os.getenv("HATHORA_API_KEY") - - payload = { - "text": text - } + payload = {"model": self._model, "text": text} if self._voice is not None: payload["voice"] = self._voice if self._speed is not None: payload["speed"] = self._speed + if self._model_config is not None: + payload["model_config"] = [ + {"name": option.name, "value": option.value} for option in self._model_config + ] yield TTSStartedFrame() async with aiohttp.ClientSession() as session: async with session.post( url, - headers={"Authorization": f"Bearer {api_key}", "Accept": "application/octet-stream"}, + headers={"Authorization": f"Bearer {self._api_key}"}, json=payload, ) as resp: audio_data = await resp.read() @@ -112,8 +128,6 @@ class KokoroTTSService(TTSService): fallback_sample_rate=self.sample_rate or self._init_sample_rate or 24000, ) - await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame( audio=pcm_audio, sample_rate=sample_rate, @@ -123,107 +137,8 @@ class KokoroTTSService(TTSService): yield frame except Exception as e: - logger.error(f"Hathora error: {e}") - yield ErrorFrame(f"Hathora error: {str(e)}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") finally: await self.stop_ttfb_metrics() await self.stop_processing_metrics() yield TTSStoppedFrame() - -class ChatterboxTTSService(TTSService): - """Chatterbox is a public text-to-speech model optimized for natural and expressive voice synthesis. - - This service uses the Hathora-hosted Chatterbox model via the HTTP API. - - [Documentation](https://models.hathora.dev/model/resemble-ai-chatterbox) - """ - - def __init__( - self, - *, - base_url = None, - api_key = None, - exaggeration: Optional[float] = None, - audio_prompt: Optional[bytes] = None, - cfg_weight: Optional[float] = None, - **kwargs, - ): - """Initialize the Hathora-hosted Chatterbox TTS service. - - Args: - base_url: Base URL for the Hathora Chatterbox TTS API. - api_key: API key for authentication with the Hathora service; - provisiion one [here](https://models.hathora.dev/tokens). - exaggeration: Controls emotional intensity (default: 0.5). - audio_prompt: Reference audio file for voice cloning. - cfg_weight: Controls adherence to reference voice (default: 0.5). - """ - - super().__init__( - **kwargs, - ) - self._base_url = base_url - self._api_key = api_key - self._exaggeration = exaggeration - self._audio_prompt = audio_prompt - self._cfg_weight = cfg_weight - - def can_generate_metrics(self) -> bool: - return True - - async def run_tts(self, text: str): - try: - await self.start_ttfb_metrics() - - url = f"{self._base_url}" - - url_query_params = [] - if self._exaggeration is not None: - url_query_params.append(f"exaggeration={self._exaggeration}") - if self._cfg_weight is not None: - url_query_params.append(f"cfg_weight={self._cfg_weight}") - - if len(url_query_params) > 0: - url += "?" + "&".join(url_query_params) - - api_key = self._api_key or os.getenv("HATHORA_API_KEY") - - form_data = aiohttp.FormData() - form_data.add_field("text", text) - - if self._audio_prompt is not None: - form_data.add_field("audio_prompt", self._audio_prompt, filename="audio.wav", content_type="application/octet-stream") - - yield TTSStartedFrame() - - async with aiohttp.ClientSession() as session: - async with session.post( - url, - headers={"Authorization": f"Bearer {api_key}"}, - data=form_data, - ) as resp: - audio_data = await resp.read() - - await self.start_tts_usage_metrics(text) - - pcm_audio, sample_rate, num_channels = _decode_audio_payload( - audio_data, - fallback_sample_rate=self.sample_rate or self._init_sample_rate or 24000, - ) - - await self.stop_ttfb_metrics() - - frame = TTSAudioRawFrame( - audio=pcm_audio, - sample_rate=sample_rate, - num_channels=num_channels, - ) - - yield frame - - except Exception as e: - logger.error(f"Hathora error: {e}") - yield ErrorFrame(f"Hathora error: {str(e)}") - finally: - await self.stop_ttfb_metrics() - yield TTSStoppedFrame() diff --git a/src/pipecat/services/hathora/utils.py b/src/pipecat/services/hathora/utils.py new file mode 100644 index 000000000..6d8eb54b8 --- /dev/null +++ b/src/pipecat/services/hathora/utils.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Utilities and types for [Hathora-hosted](https://models.hathora.dev) voice services.""" + +from dataclasses import dataclass + + +@dataclass +class ConfigOption: + """Extra configuration option passed into model_config for Hathora (if supported by model). + + Args: + name: Name of the configuration option. + value: Value of the configuration option. + """ + + name: str + value: str