transition Hathora service to use the unified API and apply PR feedback
add Hathora to root files Hathora run linter added hathora changelog
This commit is contained in:
Mike Seese
@@ -73,9 +73,9 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout
|
||||
|
||||
| Category | Services |
|
||||
| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) |
|
||||
| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [Hathora](https://docs.pipecat.ai/server/services/stt/hathora), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) |
|
||||
| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) |
|
||||
| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) |
|
||||
| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hathora](https://docs.pipecat.ai/server/services/tts/hathora), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) |
|
||||
| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), |
|
||||
| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local |
|
||||
| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) |
|
||||
|
||||
1
changelog/3169.added.md
Normal file
1
changelog/3169.added.md
Normal file
@@ -0,0 +1 @@
|
||||
- Added Hathora service to support Hathora-hosted TTS and STT models (only non-streaming)
|
||||
@@ -85,6 +85,9 @@ GROK_API_KEY=...
|
||||
# Groq
|
||||
GROQ_API_KEY=...
|
||||
|
||||
# Hathora
|
||||
HATHORA_API_KEY=...
|
||||
|
||||
# Heygen
|
||||
HEYGEN_API_KEY=...
|
||||
HEYGEN_LIVE_AVATAR_API_KEY=...
|
||||
|
||||
@@ -21,8 +21,8 @@ from pipecat.processors.aggregators.llm_context import LLMContext
|
||||
from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
|
||||
from pipecat.runner.types import RunnerArguments
|
||||
from pipecat.runner.utils import create_transport
|
||||
from pipecat.services.hathora.stt import ParakeetSTTService
|
||||
from pipecat.services.hathora.tts import ChatterboxTTSService, KokoroTTSService
|
||||
from pipecat.services.hathora.stt import HathoraSTTService
|
||||
from pipecat.services.hathora.tts import HathoraTTSService
|
||||
from pipecat.services.openai.llm import OpenAILLMService
|
||||
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
||||
from pipecat.transports.daily.transport import DailyParams
|
||||
@@ -38,13 +38,19 @@ transport_params = {
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
||||
turn_analyzer=LocalSmartTurnAnalyzerV3(),
|
||||
turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
|
||||
),
|
||||
"twilio": lambda: FastAPIWebsocketParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
||||
turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
|
||||
),
|
||||
"webrtc": lambda: TransportParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
||||
turn_analyzer=LocalSmartTurnAnalyzerV3(),
|
||||
turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
|
||||
),
|
||||
}
|
||||
|
||||
@@ -52,24 +58,14 @@ transport_params = {
|
||||
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
logger.info(f"Starting bot")
|
||||
|
||||
# See https://models.hathora.dev/model/nvidia-parakeet-tdt-0.6b-v3
|
||||
stt = ParakeetTDTSTTService(
|
||||
base_url="https://app-1c7bebb9-6977-4101-9619-833b251b86d1.app.hathora.dev/v1/transcribe",
|
||||
api_key=os.getenv("HATHORA_API_KEY")
|
||||
stt = HathoraSTTService(
|
||||
model="nvidia-parakeet-tdt-0.6b-v3",
|
||||
)
|
||||
|
||||
# See https://models.hathora.dev/model/hexgrad-kokoro-82m
|
||||
tts = KokoroTTSService(
|
||||
base_url="https://app-01312daf-6e53-4b9d-a4ad-13039f35adc4.app.hathora.dev/synthesize",
|
||||
api_key=os.getenv("HATHORA_API_KEY"),
|
||||
tts = HathoraTTSService(
|
||||
model="hexgrad-kokoro-82m",
|
||||
)
|
||||
|
||||
# See https://models.hathora.dev/model/resemble-ai-chatterbox
|
||||
# tts = ChatterboxTTSService(
|
||||
# base_url="https://app-efbc8fe2-df55-4f96-bbe3-74f6ea9d986b.app.hathora.dev/v1/generate",
|
||||
# api_key=os.getenv("HATHORA_API_KEY")
|
||||
# )
|
||||
|
||||
# See https://models.hathora.dev/model/qwen3-30b-a3b
|
||||
llm = OpenAILLMService(
|
||||
base_url="https://app-362f7ca1-6975-4e18-a605-ab202bf2c315.app.hathora.dev/v1",
|
||||
@@ -1,14 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import sys
|
||||
|
||||
from pipecat.services import DeprecatedModuleProxy
|
||||
|
||||
from .stt import *
|
||||
from .tts import *
|
||||
|
||||
sys.modules[__name__] = DeprecatedModuleProxy(globals(), "hathora", "hathora.[stt,tts]")
|
||||
|
||||
@@ -6,102 +6,126 @@
|
||||
|
||||
"""[Hathora-hosted](https://models.hathora.dev) speech-to-text services."""
|
||||
|
||||
import base64
|
||||
import os
|
||||
from typing import Optional
|
||||
from typing import AsyncGenerator, Optional
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
ErrorFrame,
|
||||
Frame,
|
||||
TranscriptionFrame,
|
||||
)
|
||||
from pipecat.services.stt_service import SegmentedSTTService
|
||||
from pipecat.transcriptions.language import Language
|
||||
from pipecat.utils.time import time_now_iso8601
|
||||
from pipecat.utils.tracing.service_decorators import traced_stt
|
||||
|
||||
class ParakeetTDTSTTService(SegmentedSTTService):
|
||||
"""Parakeet TDT is a multilingual automatic speech recognition model
|
||||
with word-level timestamps.
|
||||
from .utils import ConfigOption
|
||||
|
||||
This service uses the Hathora-hosted Parakeet model via the HTTP API.
|
||||
|
||||
[Documentation](https://models.hathora.dev/model/nvidia-parakeet-tdt-0.6b-v3)
|
||||
class HathoraSTTService(SegmentedSTTService):
|
||||
"""This service supports several different speech-to-text models hosted by Hathora.
|
||||
|
||||
[Documentation](https://models.hathora.dev)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url = None,
|
||||
api_key = None,
|
||||
start_time: Optional[int] = None,
|
||||
end_time: Optional[int] = None,
|
||||
model: str,
|
||||
language: Optional[str] = None,
|
||||
model_config: Optional[list[ConfigOption]] = None,
|
||||
base_url: str = "https://api.models.hathora.dev/inference/v1/stt",
|
||||
api_key: Optional[str] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Initialize the Hathora-hosted Parakeet STT service.
|
||||
"""Initialize the Hathora STT service.
|
||||
|
||||
Args:
|
||||
base_url: Base URL for the Hathora Parakeet STT API.
|
||||
model: Model to use; find available models
|
||||
[here](https://models.hathora.dev).
|
||||
language: Language code (if supported by model).
|
||||
model_config: Some models support additional config, refer to
|
||||
[docs](https://models.hathora.dev) for each model to see
|
||||
what is supported.
|
||||
base_url: Base API URL for the Hathora STT service.
|
||||
api_key: API key for authentication with the Hathora service;
|
||||
provisiion one [here](https://models.hathora.dev/tokens).
|
||||
start_time: Start time in seconds for the time window.
|
||||
end_time: End time in seconds for the time window.
|
||||
provision one [here](https://models.hathora.dev/tokens).
|
||||
**kwargs: Additional arguments passed to the parent class.
|
||||
"""
|
||||
super().__init__(
|
||||
**kwargs,
|
||||
)
|
||||
self._model = model
|
||||
self._language = language
|
||||
self._model_config = model_config
|
||||
self._base_url = base_url
|
||||
self._api_key = api_key
|
||||
self._start_time = start_time
|
||||
self._end_time = end_time
|
||||
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
|
||||
|
||||
def can_generate_metrics(self) -> bool:
|
||||
return True
|
||||
@traced_stt
|
||||
async def _handle_transcription(
|
||||
self, transcript: str, is_final: bool, language: Optional[Language] = None
|
||||
):
|
||||
"""Handle a transcription result with tracing."""
|
||||
pass
|
||||
|
||||
async def run_stt(self, audio: bytes):
|
||||
@traced_stt
|
||||
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
|
||||
"""Run speech-to-text on the provided audio data.
|
||||
|
||||
Args:
|
||||
audio: Raw audio bytes to transcribe.
|
||||
|
||||
Yields:
|
||||
Frame: Frames containing transcription results (typically TextFrame).
|
||||
"""
|
||||
try:
|
||||
await self.start_processing_metrics()
|
||||
await self.start_ttfb_metrics()
|
||||
|
||||
url = f"{self._base_url}"
|
||||
|
||||
url_query_params = []
|
||||
if self._start_time is not None:
|
||||
url_query_params.append(f"start_time={self._start_time}")
|
||||
if self._end_time is not None:
|
||||
url_query_params.append(f"end_time={self._end_time}")
|
||||
url_query_params.append(f"sample_rate={self.sample_rate}")
|
||||
payload = {
|
||||
"model": self._model,
|
||||
}
|
||||
|
||||
if len(url_query_params) > 0:
|
||||
url += "?" + "&".join(url_query_params)
|
||||
if self._language is not None:
|
||||
payload["language"] = self._language
|
||||
if self._model_config is not None:
|
||||
payload["model_config"] = [
|
||||
{"name": option.name, "value": option.value} for option in self._model_config
|
||||
]
|
||||
|
||||
api_key = self._api_key or os.getenv("HATHORA_API_KEY")
|
||||
|
||||
form_data = aiohttp.FormData()
|
||||
form_data.add_field("file", audio, filename="audio.wav", content_type="application/octet-stream")
|
||||
base64_audio = base64.b64encode(audio).decode("utf-8")
|
||||
payload["audio"] = base64_audio
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(
|
||||
url,
|
||||
headers={"Authorization": f"Bearer {api_key}"},
|
||||
data=form_data,
|
||||
headers={"Authorization": f"Bearer {self._api_key}"},
|
||||
json=payload,
|
||||
) as resp:
|
||||
response = await resp.json()
|
||||
|
||||
if response and "text" in response:
|
||||
text = response["text"].strip()
|
||||
if text: # Only yield non-empty text
|
||||
await self.stop_ttfb_metrics()
|
||||
await self.stop_processing_metrics()
|
||||
logger.debug(f"Transcription: [{text}]")
|
||||
# Hathora's API currently doesn't return language info
|
||||
# so we default to the requested language or "en"
|
||||
response_language = self._language or "en"
|
||||
await self._handle_transcription(text, True, response_language)
|
||||
yield TranscriptionFrame(
|
||||
text,
|
||||
self._user_id,
|
||||
time_now_iso8601(),
|
||||
Language("en"), # TODO: the parakeet hathora API doesn't accept a language but says it's multilingual
|
||||
Language(response_language),
|
||||
result=response,
|
||||
)
|
||||
|
||||
await self.stop_ttfb_metrics()
|
||||
await self.stop_processing_metrics()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Hathora error: {e}")
|
||||
yield ErrorFrame(f"Hathora error: {str(e)}")
|
||||
yield ErrorFrame(error=f"Unknown error occurred: {e}")
|
||||
|
||||
@@ -9,18 +9,22 @@
|
||||
import io
|
||||
import os
|
||||
import wave
|
||||
from typing import Optional, Tuple
|
||||
from typing import AsyncGenerator, Optional, Tuple
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
ErrorFrame,
|
||||
Frame,
|
||||
TTSAudioRawFrame,
|
||||
TTSStartedFrame,
|
||||
TTSStoppedFrame,
|
||||
)
|
||||
from pipecat.services.tts_service import TTSService
|
||||
from pipecat.utils.tracing.service_decorators import traced_tts
|
||||
|
||||
from .utils import ConfigOption
|
||||
|
||||
|
||||
def _decode_audio_payload(
|
||||
audio_bytes: bytes,
|
||||
@@ -29,7 +33,6 @@ def _decode_audio_payload(
|
||||
fallback_channels: int = 1,
|
||||
) -> Tuple[bytes, int, int]:
|
||||
"""Convert a WAV/PCM payload into raw PCM samples for TTSAudioRawFrame."""
|
||||
|
||||
try:
|
||||
with wave.open(io.BytesIO(audio_bytes), "rb") as wav_reader:
|
||||
channels = wav_reader.getnchannels()
|
||||
@@ -40,69 +43,82 @@ def _decode_audio_payload(
|
||||
# If the payload is already raw PCM, just pass it through.
|
||||
return audio_bytes, fallback_sample_rate, fallback_channels
|
||||
|
||||
class KokoroTTSService(TTSService):
|
||||
"""Kokoro is an open-weight TTS model with 82 million parameters.
|
||||
|
||||
This service uses the Hathora-hosted Kokoro model via the HTTP API.
|
||||
class HathoraTTSService(TTSService):
|
||||
"""This service supports several different text-to-speech models hosted by Hathora.
|
||||
|
||||
[Documentation](https://models.hathora.dev/model/hexgrad-kokoro-82m)
|
||||
[Documentation](https://models.hathora.dev)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url = None,
|
||||
api_key = None,
|
||||
model: str,
|
||||
voice: Optional[str] = None,
|
||||
speed: Optional[float] = None,
|
||||
model_config: Optional[list[ConfigOption]] = None,
|
||||
base_url: str = "https://api.models.hathora.dev/inference/v1/tts",
|
||||
api_key: Optional[str] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Initialize the Hathora-hosted Kokoro TTS service.
|
||||
"""Initialize the Hathora TTS service.
|
||||
|
||||
Args:
|
||||
base_url: Base URL for the Hathora Kokoro TTS API, .
|
||||
model: Model to use; find available models
|
||||
[here](https://models.hathora.dev).
|
||||
voice: Voice to use for synthesis (if supported by model).
|
||||
speed: Speech speed multiplier (if supported by model).
|
||||
model_config: Some models support additional config, refer to
|
||||
[docs](https://models.hathora.dev) for each model to see
|
||||
what is supported.
|
||||
base_url: Base API URL for the Hathora TTS service.
|
||||
api_key: API key for authentication with the Hathora service;
|
||||
provisiion one [here](https://models.hathora.dev/tokens).
|
||||
voice: Voice to use for synthesis (see the
|
||||
[Hathora docs](https://models.hathora.dev/model/hexgrad-kokoro-82m)
|
||||
for the default value; [list of voices](https://huggingface.co/hexgrad/Kokoro-82M/blob/main/VOICES.md)).
|
||||
speed: Speech speed multiplier (0.5 = half speed, 2.0 = double speed, default: 1).
|
||||
provision one [here](https://models.hathora.dev/tokens).
|
||||
**kwargs: Additional arguments passed to the parent class.
|
||||
"""
|
||||
super().__init__(
|
||||
**kwargs,
|
||||
)
|
||||
self._base_url = base_url
|
||||
self._api_key = api_key
|
||||
self._model = model
|
||||
self._voice = voice
|
||||
self._speed = speed
|
||||
self._model_config = model_config
|
||||
self._base_url = base_url
|
||||
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
|
||||
|
||||
def can_generate_metrics(self) -> bool:
|
||||
return True
|
||||
@traced_tts
|
||||
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
|
||||
"""Run text-to-speech synthesis on the provided text.
|
||||
|
||||
async def run_tts(self, text: str):
|
||||
Args:
|
||||
text: The text to synthesize into speech.
|
||||
|
||||
Yields:
|
||||
Frame: Audio frames containing the synthesized speech.
|
||||
"""
|
||||
try:
|
||||
await self.start_processing_metrics()
|
||||
await self.start_ttfb_metrics()
|
||||
|
||||
url = f"{self._base_url}"
|
||||
|
||||
api_key = self._api_key or os.getenv("HATHORA_API_KEY")
|
||||
|
||||
payload = {
|
||||
"text": text
|
||||
}
|
||||
payload = {"model": self._model, "text": text}
|
||||
|
||||
if self._voice is not None:
|
||||
payload["voice"] = self._voice
|
||||
if self._speed is not None:
|
||||
payload["speed"] = self._speed
|
||||
if self._model_config is not None:
|
||||
payload["model_config"] = [
|
||||
{"name": option.name, "value": option.value} for option in self._model_config
|
||||
]
|
||||
|
||||
yield TTSStartedFrame()
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(
|
||||
url,
|
||||
headers={"Authorization": f"Bearer {api_key}", "Accept": "application/octet-stream"},
|
||||
headers={"Authorization": f"Bearer {self._api_key}"},
|
||||
json=payload,
|
||||
) as resp:
|
||||
audio_data = await resp.read()
|
||||
@@ -112,8 +128,6 @@ class KokoroTTSService(TTSService):
|
||||
fallback_sample_rate=self.sample_rate or self._init_sample_rate or 24000,
|
||||
)
|
||||
|
||||
await self.stop_ttfb_metrics()
|
||||
|
||||
frame = TTSAudioRawFrame(
|
||||
audio=pcm_audio,
|
||||
sample_rate=sample_rate,
|
||||
@@ -123,107 +137,8 @@ class KokoroTTSService(TTSService):
|
||||
yield frame
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Hathora error: {e}")
|
||||
yield ErrorFrame(f"Hathora error: {str(e)}")
|
||||
yield ErrorFrame(error=f"Unknown error occurred: {e}")
|
||||
finally:
|
||||
await self.stop_ttfb_metrics()
|
||||
await self.stop_processing_metrics()
|
||||
yield TTSStoppedFrame()
|
||||
|
||||
class ChatterboxTTSService(TTSService):
|
||||
"""Chatterbox is a public text-to-speech model optimized for natural and expressive voice synthesis.
|
||||
|
||||
This service uses the Hathora-hosted Chatterbox model via the HTTP API.
|
||||
|
||||
[Documentation](https://models.hathora.dev/model/resemble-ai-chatterbox)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url = None,
|
||||
api_key = None,
|
||||
exaggeration: Optional[float] = None,
|
||||
audio_prompt: Optional[bytes] = None,
|
||||
cfg_weight: Optional[float] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Initialize the Hathora-hosted Chatterbox TTS service.
|
||||
|
||||
Args:
|
||||
base_url: Base URL for the Hathora Chatterbox TTS API.
|
||||
api_key: API key for authentication with the Hathora service;
|
||||
provisiion one [here](https://models.hathora.dev/tokens).
|
||||
exaggeration: Controls emotional intensity (default: 0.5).
|
||||
audio_prompt: Reference audio file for voice cloning.
|
||||
cfg_weight: Controls adherence to reference voice (default: 0.5).
|
||||
"""
|
||||
|
||||
super().__init__(
|
||||
**kwargs,
|
||||
)
|
||||
self._base_url = base_url
|
||||
self._api_key = api_key
|
||||
self._exaggeration = exaggeration
|
||||
self._audio_prompt = audio_prompt
|
||||
self._cfg_weight = cfg_weight
|
||||
|
||||
def can_generate_metrics(self) -> bool:
|
||||
return True
|
||||
|
||||
async def run_tts(self, text: str):
|
||||
try:
|
||||
await self.start_ttfb_metrics()
|
||||
|
||||
url = f"{self._base_url}"
|
||||
|
||||
url_query_params = []
|
||||
if self._exaggeration is not None:
|
||||
url_query_params.append(f"exaggeration={self._exaggeration}")
|
||||
if self._cfg_weight is not None:
|
||||
url_query_params.append(f"cfg_weight={self._cfg_weight}")
|
||||
|
||||
if len(url_query_params) > 0:
|
||||
url += "?" + "&".join(url_query_params)
|
||||
|
||||
api_key = self._api_key or os.getenv("HATHORA_API_KEY")
|
||||
|
||||
form_data = aiohttp.FormData()
|
||||
form_data.add_field("text", text)
|
||||
|
||||
if self._audio_prompt is not None:
|
||||
form_data.add_field("audio_prompt", self._audio_prompt, filename="audio.wav", content_type="application/octet-stream")
|
||||
|
||||
yield TTSStartedFrame()
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(
|
||||
url,
|
||||
headers={"Authorization": f"Bearer {api_key}"},
|
||||
data=form_data,
|
||||
) as resp:
|
||||
audio_data = await resp.read()
|
||||
|
||||
await self.start_tts_usage_metrics(text)
|
||||
|
||||
pcm_audio, sample_rate, num_channels = _decode_audio_payload(
|
||||
audio_data,
|
||||
fallback_sample_rate=self.sample_rate or self._init_sample_rate or 24000,
|
||||
)
|
||||
|
||||
await self.stop_ttfb_metrics()
|
||||
|
||||
frame = TTSAudioRawFrame(
|
||||
audio=pcm_audio,
|
||||
sample_rate=sample_rate,
|
||||
num_channels=num_channels,
|
||||
)
|
||||
|
||||
yield frame
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Hathora error: {e}")
|
||||
yield ErrorFrame(f"Hathora error: {str(e)}")
|
||||
finally:
|
||||
await self.stop_ttfb_metrics()
|
||||
yield TTSStoppedFrame()
|
||||
|
||||
22
src/pipecat/services/hathora/utils.py
Normal file
22
src/pipecat/services/hathora/utils.py
Normal file
@@ -0,0 +1,22 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
"""Utilities and types for [Hathora-hosted](https://models.hathora.dev) voice services."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConfigOption:
|
||||
"""Extra configuration option passed into model_config for Hathora (if supported by model).
|
||||
|
||||
Args:
|
||||
name: Name of the configuration option.
|
||||
value: Value of the configuration option.
|
||||
"""
|
||||
|
||||
name: str
|
||||
value: str
|
||||
Reference in New Issue
Block a user