Merge pull request #3169 from Incanta/hathora
Add Hathora STT and TTS services
This commit is contained in:
@@ -73,9 +73,9 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout
|
|||||||
|
|
||||||
| Category | Services |
|
| Category | Services |
|
||||||
| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) |
|
| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [Hathora](https://docs.pipecat.ai/server/services/stt/hathora), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) |
|
||||||
| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) |
|
| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) |
|
||||||
| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) |
|
| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hathora](https://docs.pipecat.ai/server/services/tts/hathora), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) |
|
||||||
| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), |
|
| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), |
|
||||||
| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local |
|
| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local |
|
||||||
| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) |
|
| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) |
|
||||||
|
|||||||
1
changelog/3169.added.md
Normal file
1
changelog/3169.added.md
Normal file
@@ -0,0 +1 @@
|
|||||||
|
- Added Hathora service to support Hathora-hosted TTS and STT models (only non-streaming)
|
||||||
@@ -85,6 +85,9 @@ GROK_API_KEY=...
|
|||||||
# Groq
|
# Groq
|
||||||
GROQ_API_KEY=...
|
GROQ_API_KEY=...
|
||||||
|
|
||||||
|
# Hathora
|
||||||
|
HATHORA_API_KEY=...
|
||||||
|
|
||||||
# Heygen
|
# Heygen
|
||||||
HEYGEN_API_KEY=...
|
HEYGEN_API_KEY=...
|
||||||
HEYGEN_LIVE_AVATAR_API_KEY=...
|
HEYGEN_LIVE_AVATAR_API_KEY=...
|
||||||
|
|||||||
141
examples/foundational/07zh-interruptible-hathora.py
Normal file
141
examples/foundational/07zh-interruptible-hathora.py
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
|
||||||
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
|
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||||
|
from pipecat.frames.frames import LLMRunFrame
|
||||||
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||||
|
from pipecat.processors.aggregators.llm_response_universal import (
|
||||||
|
LLMContextAggregatorPair,
|
||||||
|
LLMUserAggregatorParams,
|
||||||
|
)
|
||||||
|
from pipecat.runner.types import RunnerArguments
|
||||||
|
from pipecat.runner.utils import create_transport
|
||||||
|
from pipecat.services.hathora.stt import HathoraSTTService
|
||||||
|
from pipecat.services.hathora.tts import HathoraTTSService
|
||||||
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
|
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
||||||
|
from pipecat.transports.daily.transport import DailyParams
|
||||||
|
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
|
||||||
|
from pipecat.turns.user_stop import TurnAnalyzerUserTurnStopStrategy
|
||||||
|
from pipecat.turns.user_turn_strategies import UserTurnStrategies
|
||||||
|
|
||||||
|
load_dotenv(override=True)
|
||||||
|
|
||||||
|
# We store functions so objects (e.g. SileroVADAnalyzer) don't get
|
||||||
|
# instantiated. The function will be called when the desired transport gets
|
||||||
|
# selected.
|
||||||
|
transport_params = {
|
||||||
|
"daily": lambda: DailyParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
|
audio_out_enabled=True,
|
||||||
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
||||||
|
),
|
||||||
|
"twilio": lambda: FastAPIWebsocketParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
|
audio_out_enabled=True,
|
||||||
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
||||||
|
),
|
||||||
|
"webrtc": lambda: TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
|
audio_out_enabled=True,
|
||||||
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
|
stt = HathoraSTTService(
|
||||||
|
model="nvidia-parakeet-tdt-0.6b-v3",
|
||||||
|
)
|
||||||
|
|
||||||
|
tts = HathoraTTSService(
|
||||||
|
model="hexgrad-kokoro-82m",
|
||||||
|
)
|
||||||
|
|
||||||
|
# See https://models.hathora.dev/model/qwen3-30b-a3b
|
||||||
|
llm = OpenAILLMService(
|
||||||
|
base_url="https://app-362f7ca1-6975-4e18-a605-ab202bf2c315.app.hathora.dev/v1",
|
||||||
|
api_key=os.getenv("HATHORA_API_KEY"),
|
||||||
|
model=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
context = LLMContext(messages)
|
||||||
|
context_aggregator = LLMContextAggregatorPair(
|
||||||
|
context,
|
||||||
|
user_params=LLMUserAggregatorParams(
|
||||||
|
user_turn_strategies=UserTurnStrategies(
|
||||||
|
stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3())]
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
pipeline = Pipeline(
|
||||||
|
[
|
||||||
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
|
context_aggregator.user(), # User responses
|
||||||
|
llm, # LLM
|
||||||
|
tts, # TTS
|
||||||
|
transport.output(), # Transport bot output
|
||||||
|
context_aggregator.assistant(), # Assistant spoken responses
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
task = PipelineTask(
|
||||||
|
pipeline,
|
||||||
|
params=PipelineParams(
|
||||||
|
enable_metrics=True,
|
||||||
|
enable_usage_metrics=True,
|
||||||
|
),
|
||||||
|
idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
|
||||||
|
)
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
# Kick off the conversation.
|
||||||
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
|
await task.queue_frames([LLMRunFrame()])
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
||||||
|
|
||||||
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
|
async def bot(runner_args: RunnerArguments):
|
||||||
|
"""Main bot entry point compatible with Pipecat Cloud."""
|
||||||
|
transport = await create_transport(runner_args, transport_params)
|
||||||
|
await run_bot(transport, runner_args)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
from pipecat.runner.run import main
|
||||||
|
|
||||||
|
main()
|
||||||
@@ -137,6 +137,7 @@ TESTS_07 = [
|
|||||||
# ("07zd-interruptible-aicoustics.py", EVAL_SIMPLE_MATH),
|
# ("07zd-interruptible-aicoustics.py", EVAL_SIMPLE_MATH),
|
||||||
("07ze-interruptible-hume.py", EVAL_SIMPLE_MATH),
|
("07ze-interruptible-hume.py", EVAL_SIMPLE_MATH),
|
||||||
("07zf-interruptible-gradium.py", EVAL_SIMPLE_MATH),
|
("07zf-interruptible-gradium.py", EVAL_SIMPLE_MATH),
|
||||||
|
("07zh-interruptible-hathora.py", EVAL_SIMPLE_MATH),
|
||||||
# Needs a local XTTS docker instance running.
|
# Needs a local XTTS docker instance running.
|
||||||
# ("07i-interruptible-xtts.py", EVAL_SIMPLE_MATH),
|
# ("07i-interruptible-xtts.py", EVAL_SIMPLE_MATH),
|
||||||
# Needs a Krisp license.
|
# Needs a Krisp license.
|
||||||
|
|||||||
0
src/pipecat/services/hathora/__init__.py
Normal file
0
src/pipecat/services/hathora/__init__.py
Normal file
160
src/pipecat/services/hathora/stt.py
Normal file
160
src/pipecat/services/hathora/stt.py
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
"""[Hathora-hosted](https://models.hathora.dev) speech-to-text services."""
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import os
|
||||||
|
from typing import AsyncGenerator, Optional
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from pipecat.frames.frames import (
|
||||||
|
ErrorFrame,
|
||||||
|
Frame,
|
||||||
|
TranscriptionFrame,
|
||||||
|
)
|
||||||
|
from pipecat.services.stt_service import SegmentedSTTService
|
||||||
|
from pipecat.transcriptions.language import Language
|
||||||
|
from pipecat.utils.time import time_now_iso8601
|
||||||
|
from pipecat.utils.tracing.service_decorators import traced_stt
|
||||||
|
|
||||||
|
from .utils import ConfigOption
|
||||||
|
|
||||||
|
|
||||||
|
class HathoraSTTService(SegmentedSTTService):
|
||||||
|
"""This service supports several different speech-to-text models hosted by Hathora.
|
||||||
|
|
||||||
|
[Documentation](https://models.hathora.dev)
|
||||||
|
"""
|
||||||
|
|
||||||
|
class InputParams(BaseModel):
|
||||||
|
"""Optional input parameters for Hathora STT configuration.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
language: Language code (if supported by model).
|
||||||
|
config: Some models support additional config, refer to
|
||||||
|
[docs](https://models.hathora.dev) for each model to see
|
||||||
|
what is supported.
|
||||||
|
"""
|
||||||
|
|
||||||
|
language: Optional[str] = None
|
||||||
|
config: Optional[list[ConfigOption]] = None
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
model: str,
|
||||||
|
sample_rate: Optional[int] = None,
|
||||||
|
api_key: Optional[str] = None,
|
||||||
|
base_url: str = "https://api.models.hathora.dev/inference/v1/stt",
|
||||||
|
params: Optional[InputParams] = None,
|
||||||
|
**kwargs,
|
||||||
|
):
|
||||||
|
"""Initialize the Hathora STT service.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
model: Model to use; find available models
|
||||||
|
[here](https://models.hathora.dev).
|
||||||
|
sample_rate: The sample rate for audio input. If None, will be determined
|
||||||
|
from the start frame.
|
||||||
|
api_key: API key for authentication with the Hathora service;
|
||||||
|
provision one [here](https://models.hathora.dev/tokens).
|
||||||
|
base_url: Base API URL for the Hathora STT service.
|
||||||
|
params: Configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to the parent class.
|
||||||
|
"""
|
||||||
|
super().__init__(
|
||||||
|
sample_rate=sample_rate,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
self._model = model
|
||||||
|
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
|
||||||
|
self._base_url = base_url
|
||||||
|
|
||||||
|
params = params or HathoraSTTService.InputParams()
|
||||||
|
|
||||||
|
self._settings = {
|
||||||
|
"language": params.language,
|
||||||
|
"config": params.config,
|
||||||
|
}
|
||||||
|
|
||||||
|
self.set_model_name(model)
|
||||||
|
|
||||||
|
def can_generate_metrics(self) -> bool:
|
||||||
|
"""Check if this service can generate processing metrics.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True
|
||||||
|
"""
|
||||||
|
return True
|
||||||
|
|
||||||
|
@traced_stt
|
||||||
|
async def _handle_transcription(
|
||||||
|
self, transcript: str, is_final: bool, language: Optional[Language] = None
|
||||||
|
):
|
||||||
|
"""Handle a transcription result with tracing."""
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
|
||||||
|
"""Run speech-to-text on the provided audio data.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Raw audio bytes to transcribe.
|
||||||
|
|
||||||
|
Yields:
|
||||||
|
Frame: Frames containing transcription results (typically TextFrame).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
await self.start_processing_metrics()
|
||||||
|
await self.start_ttfb_metrics()
|
||||||
|
|
||||||
|
url = f"{self._base_url}"
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"model": self._model,
|
||||||
|
}
|
||||||
|
|
||||||
|
if self._settings["language"] is not None:
|
||||||
|
payload["language"] = self._settings["language"]
|
||||||
|
if self._settings["config"] is not None:
|
||||||
|
payload["model_config"] = [
|
||||||
|
{"name": option.name, "value": option.value}
|
||||||
|
for option in self._settings["config"]
|
||||||
|
]
|
||||||
|
|
||||||
|
base64_audio = base64.b64encode(audio).decode("utf-8")
|
||||||
|
payload["audio"] = base64_audio
|
||||||
|
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
async with session.post(
|
||||||
|
url,
|
||||||
|
headers={"Authorization": f"Bearer {self._api_key}"},
|
||||||
|
json=payload,
|
||||||
|
) as resp:
|
||||||
|
response = await resp.json()
|
||||||
|
|
||||||
|
if response and "text" in response:
|
||||||
|
text = response["text"].strip()
|
||||||
|
if text: # Only yield non-empty text
|
||||||
|
# Hathora's API currently doesn't return language info
|
||||||
|
# so we default to the requested language or "en"
|
||||||
|
response_language = self._settings["language"] or "en"
|
||||||
|
await self._handle_transcription(text, True, response_language)
|
||||||
|
yield TranscriptionFrame(
|
||||||
|
text,
|
||||||
|
self._user_id,
|
||||||
|
time_now_iso8601(),
|
||||||
|
Language(response_language),
|
||||||
|
result=response,
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.stop_ttfb_metrics()
|
||||||
|
await self.stop_processing_metrics()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
yield ErrorFrame(error=f"Unknown error occurred: {e}")
|
||||||
173
src/pipecat/services/hathora/tts.py
Normal file
173
src/pipecat/services/hathora/tts.py
Normal file
@@ -0,0 +1,173 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
"""[Hathora-hosted](https://models.hathora.dev) text-to-speech services."""
|
||||||
|
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import wave
|
||||||
|
from typing import AsyncGenerator, Optional, Tuple
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from pipecat.frames.frames import (
|
||||||
|
ErrorFrame,
|
||||||
|
Frame,
|
||||||
|
TTSAudioRawFrame,
|
||||||
|
TTSStartedFrame,
|
||||||
|
TTSStoppedFrame,
|
||||||
|
)
|
||||||
|
from pipecat.services.tts_service import TTSService
|
||||||
|
from pipecat.utils.tracing.service_decorators import traced_tts
|
||||||
|
|
||||||
|
from .utils import ConfigOption
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_audio_payload(
|
||||||
|
audio_bytes: bytes,
|
||||||
|
*,
|
||||||
|
fallback_sample_rate: int = 24000,
|
||||||
|
fallback_channels: int = 1,
|
||||||
|
) -> Tuple[bytes, int, int]:
|
||||||
|
"""Convert a WAV/PCM payload into raw PCM samples for TTSAudioRawFrame."""
|
||||||
|
try:
|
||||||
|
with wave.open(io.BytesIO(audio_bytes), "rb") as wav_reader:
|
||||||
|
channels = wav_reader.getnchannels()
|
||||||
|
sample_rate = wav_reader.getframerate()
|
||||||
|
frames = wav_reader.readframes(wav_reader.getnframes())
|
||||||
|
return frames, sample_rate, channels
|
||||||
|
except (wave.Error, EOFError):
|
||||||
|
# If the payload is already raw PCM, just pass it through.
|
||||||
|
return audio_bytes, fallback_sample_rate, fallback_channels
|
||||||
|
|
||||||
|
|
||||||
|
class HathoraTTSService(TTSService):
|
||||||
|
"""This service supports several different text-to-speech models hosted by Hathora.
|
||||||
|
|
||||||
|
[Documentation](https://models.hathora.dev)
|
||||||
|
"""
|
||||||
|
|
||||||
|
class InputParams(BaseModel):
|
||||||
|
"""Optional input parameters for Hathora TTS configuration.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
speed: Speech speed multiplier (if supported by model).
|
||||||
|
config: Some models support additional config, refer to
|
||||||
|
[docs](https://models.hathora.dev) for each model to see
|
||||||
|
what is supported.
|
||||||
|
"""
|
||||||
|
|
||||||
|
speed: Optional[float] = None
|
||||||
|
config: Optional[list[ConfigOption]] = None
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
model: str,
|
||||||
|
voice_id: Optional[str] = None,
|
||||||
|
sample_rate: Optional[int] = None,
|
||||||
|
api_key: Optional[str] = None,
|
||||||
|
base_url: str = "https://api.models.hathora.dev/inference/v1/tts",
|
||||||
|
params: Optional[InputParams] = None,
|
||||||
|
**kwargs,
|
||||||
|
):
|
||||||
|
"""Initialize the Hathora TTS service.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
model: Model to use; find available models
|
||||||
|
[here](https://models.hathora.dev).
|
||||||
|
voice_id: Voice to use for synthesis (if supported by model).
|
||||||
|
sample_rate: Output sample rate for generated audio.
|
||||||
|
api_key: API key for authentication with the Hathora service;
|
||||||
|
provision one [here](https://models.hathora.dev/tokens).
|
||||||
|
base_url: Base API URL for the Hathora TTS service.
|
||||||
|
params: Configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to the parent class.
|
||||||
|
"""
|
||||||
|
super().__init__(
|
||||||
|
sample_rate=sample_rate,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
self._model = model
|
||||||
|
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
|
||||||
|
self._base_url = base_url
|
||||||
|
|
||||||
|
params = params or HathoraTTSService.InputParams()
|
||||||
|
|
||||||
|
self._settings = {
|
||||||
|
"speed": params.speed,
|
||||||
|
"config": params.config,
|
||||||
|
}
|
||||||
|
|
||||||
|
self.set_model_name(model)
|
||||||
|
self.set_voice(voice_id)
|
||||||
|
|
||||||
|
def can_generate_metrics(self) -> bool:
|
||||||
|
"""Check if this service can generate processing metrics.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True
|
||||||
|
"""
|
||||||
|
return True
|
||||||
|
|
||||||
|
@traced_tts
|
||||||
|
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
|
||||||
|
"""Run text-to-speech synthesis on the provided text.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: The text to synthesize into speech.
|
||||||
|
|
||||||
|
Yields:
|
||||||
|
Frame: Audio frames containing the synthesized speech.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
await self.start_processing_metrics()
|
||||||
|
await self.start_ttfb_metrics()
|
||||||
|
|
||||||
|
url = f"{self._base_url}"
|
||||||
|
|
||||||
|
payload = {"model": self._model, "text": text}
|
||||||
|
|
||||||
|
if self._voice_id is not None:
|
||||||
|
payload["voice"] = self._voice_id
|
||||||
|
if self._settings["speed"] is not None:
|
||||||
|
payload["speed"] = self._settings["speed"]
|
||||||
|
if self._settings["config"] is not None:
|
||||||
|
payload["model_config"] = [
|
||||||
|
{"name": option.name, "value": option.value}
|
||||||
|
for option in self._settings["config"]
|
||||||
|
]
|
||||||
|
|
||||||
|
yield TTSStartedFrame()
|
||||||
|
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
async with session.post(
|
||||||
|
url,
|
||||||
|
headers={"Authorization": f"Bearer {self._api_key}"},
|
||||||
|
json=payload,
|
||||||
|
) as resp:
|
||||||
|
audio_data = await resp.read()
|
||||||
|
|
||||||
|
pcm_audio, sample_rate, num_channels = _decode_audio_payload(
|
||||||
|
audio_data,
|
||||||
|
fallback_sample_rate=self.sample_rate,
|
||||||
|
)
|
||||||
|
|
||||||
|
frame = TTSAudioRawFrame(
|
||||||
|
audio=pcm_audio,
|
||||||
|
sample_rate=self.sample_rate,
|
||||||
|
num_channels=num_channels,
|
||||||
|
)
|
||||||
|
|
||||||
|
yield frame
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
yield ErrorFrame(error=f"Unknown error occurred: {e}")
|
||||||
|
finally:
|
||||||
|
await self.stop_ttfb_metrics()
|
||||||
|
await self.stop_processing_metrics()
|
||||||
|
yield TTSStoppedFrame()
|
||||||
22
src/pipecat/services/hathora/utils.py
Normal file
22
src/pipecat/services/hathora/utils.py
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Utilities and types for [Hathora-hosted](https://models.hathora.dev) voice services."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ConfigOption:
|
||||||
|
"""Extra configuration option passed into model_config for Hathora (if supported by model).
|
||||||
|
|
||||||
|
Args:
|
||||||
|
name: Name of the configuration option.
|
||||||
|
value: Value of the configuration option.
|
||||||
|
"""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
value: str
|
||||||
Reference in New Issue
Block a user