1121 lines
45 KiB
Python
1121 lines
45 KiB
Python
#
|
|
# Copyright (c) 2024-2026, Daily
|
|
#
|
|
# SPDX-License-Identifier: BSD 2-Clause License
|
|
#
|
|
|
|
"""ElevenLabs text-to-speech service implementations.
|
|
|
|
This module provides WebSocket and HTTP-based TTS services using ElevenLabs API
|
|
with support for streaming audio, word timestamps, and voice customization.
|
|
"""
|
|
|
|
import asyncio
|
|
import base64
|
|
import json
|
|
import uuid
|
|
from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Tuple, Union
|
|
|
|
import aiohttp
|
|
from loguru import logger
|
|
from pydantic import BaseModel
|
|
|
|
from pipecat.frames.frames import (
|
|
CancelFrame,
|
|
EndFrame,
|
|
ErrorFrame,
|
|
Frame,
|
|
InterruptionFrame,
|
|
LLMFullResponseEndFrame,
|
|
StartFrame,
|
|
TTSAudioRawFrame,
|
|
TTSStartedFrame,
|
|
TTSStoppedFrame,
|
|
)
|
|
from pipecat.processors.frame_processor import FrameDirection
|
|
from pipecat.services.tts_service import (
|
|
AudioContextWordTTSService,
|
|
WordTTSService,
|
|
)
|
|
from pipecat.transcriptions.language import Language, resolve_language
|
|
from pipecat.utils.tracing.service_decorators import traced_tts
|
|
|
|
# See .env.example for ElevenLabs configuration needed
|
|
try:
|
|
import websockets
|
|
from websockets.asyncio.client import connect as websocket_connect
|
|
from websockets.protocol import State
|
|
except ModuleNotFoundError as e:
|
|
logger.error(f"Exception: {e}")
|
|
logger.error("In order to use ElevenLabs, you need to `pip install pipecat-ai[elevenlabs]`.")
|
|
raise Exception(f"Missing module: {e}")
|
|
|
|
ElevenLabsOutputFormat = Literal["pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100"]
|
|
|
|
# Models that support language codes
|
|
# The following models are excluded as they don't support language codes:
|
|
# - eleven_flash_v2
|
|
# - eleven_turbo_v2
|
|
# - eleven_multilingual_v2
|
|
ELEVENLABS_MULTILINGUAL_MODELS = {
|
|
"eleven_flash_v2_5",
|
|
"eleven_turbo_v2_5",
|
|
}
|
|
|
|
|
|
def language_to_elevenlabs_language(language: Language) -> Optional[str]:
|
|
"""Convert a Language enum to ElevenLabs language code.
|
|
|
|
Args:
|
|
language: The Language enum value to convert.
|
|
|
|
Returns:
|
|
The corresponding ElevenLabs language code, or None if not supported.
|
|
"""
|
|
LANGUAGE_MAP = {
|
|
Language.AR: "ar",
|
|
Language.BG: "bg",
|
|
Language.CS: "cs",
|
|
Language.DA: "da",
|
|
Language.DE: "de",
|
|
Language.EL: "el",
|
|
Language.EN: "en",
|
|
Language.ES: "es",
|
|
Language.FI: "fi",
|
|
Language.FIL: "fil",
|
|
Language.FR: "fr",
|
|
Language.HI: "hi",
|
|
Language.HR: "hr",
|
|
Language.HU: "hu",
|
|
Language.ID: "id",
|
|
Language.IT: "it",
|
|
Language.JA: "ja",
|
|
Language.KO: "ko",
|
|
Language.MS: "ms",
|
|
Language.NL: "nl",
|
|
Language.NO: "no",
|
|
Language.PL: "pl",
|
|
Language.PT: "pt",
|
|
Language.RO: "ro",
|
|
Language.RU: "ru",
|
|
Language.SK: "sk",
|
|
Language.SV: "sv",
|
|
Language.TA: "ta",
|
|
Language.TR: "tr",
|
|
Language.UK: "uk",
|
|
Language.VI: "vi",
|
|
Language.ZH: "zh",
|
|
}
|
|
|
|
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
|
|
|
|
|
|
def output_format_from_sample_rate(sample_rate: int) -> str:
|
|
"""Get the appropriate output format string for a given sample rate.
|
|
|
|
Args:
|
|
sample_rate: The audio sample rate in Hz.
|
|
|
|
Returns:
|
|
The ElevenLabs output format string.
|
|
"""
|
|
match sample_rate:
|
|
case 8000:
|
|
return "pcm_8000"
|
|
case 16000:
|
|
return "pcm_16000"
|
|
case 22050:
|
|
return "pcm_22050"
|
|
case 24000:
|
|
return "pcm_24000"
|
|
case 44100:
|
|
return "pcm_44100"
|
|
logger.warning(
|
|
f"ElevenLabsTTSService: No output format available for {sample_rate} sample rate"
|
|
)
|
|
return "pcm_24000"
|
|
|
|
|
|
def build_elevenlabs_voice_settings(
|
|
settings: Dict[str, Any],
|
|
) -> Optional[Dict[str, Union[float, bool]]]:
|
|
"""Build voice settings dictionary for ElevenLabs based on provided settings.
|
|
|
|
Args:
|
|
settings: Dictionary containing voice settings parameters.
|
|
|
|
Returns:
|
|
Dictionary of voice settings or None if no valid settings are provided.
|
|
"""
|
|
voice_setting_keys = ["stability", "similarity_boost", "style", "use_speaker_boost", "speed"]
|
|
|
|
voice_settings = {}
|
|
for key in voice_setting_keys:
|
|
if key in settings and settings[key] is not None:
|
|
voice_settings[key] = settings[key]
|
|
|
|
return voice_settings or None
|
|
|
|
|
|
class PronunciationDictionaryLocator(BaseModel):
|
|
"""Locator for a pronunciation dictionary.
|
|
|
|
Parameters:
|
|
pronunciation_dictionary_id: The ID of the pronunciation dictionary.
|
|
version_id: The version ID of the pronunciation dictionary.
|
|
"""
|
|
|
|
pronunciation_dictionary_id: str
|
|
version_id: str
|
|
|
|
|
|
def calculate_word_times(
|
|
alignment_info: Mapping[str, Any],
|
|
cumulative_time: float,
|
|
partial_word: str = "",
|
|
partial_word_start_time: float = 0.0,
|
|
) -> tuple[List[Tuple[str, float]], str, float]:
|
|
"""Calculate word timestamps from character alignment information.
|
|
|
|
Args:
|
|
alignment_info: Character alignment data from ElevenLabs API.
|
|
cumulative_time: Base time offset for this chunk.
|
|
partial_word: Partial word carried over from previous chunk.
|
|
partial_word_start_time: Start time of the partial word.
|
|
|
|
Returns:
|
|
Tuple of (word_times, new_partial_word, new_partial_word_start_time):
|
|
- word_times: List of (word, timestamp) tuples for complete words
|
|
- new_partial_word: Incomplete word at end of chunk (empty if chunk ends with space)
|
|
- new_partial_word_start_time: Start time of the incomplete word
|
|
"""
|
|
chars = alignment_info["chars"]
|
|
char_start_times_ms = alignment_info["charStartTimesMs"]
|
|
|
|
if len(chars) != len(char_start_times_ms):
|
|
logger.error(
|
|
f"calculate_word_times: length mismatch - chars={len(chars)}, times={len(char_start_times_ms)}"
|
|
)
|
|
return ([], partial_word, partial_word_start_time)
|
|
|
|
# Build words and track their start positions
|
|
words = []
|
|
word_start_times = []
|
|
current_word = partial_word # Start with any partial word from previous chunk
|
|
word_start_time = partial_word_start_time if partial_word else None
|
|
|
|
for i, char in enumerate(chars):
|
|
if char == " ":
|
|
# End of current word
|
|
if current_word: # Only add non-empty words
|
|
words.append(current_word)
|
|
word_start_times.append(word_start_time)
|
|
current_word = ""
|
|
word_start_time = None
|
|
else:
|
|
# Building a word
|
|
if word_start_time is None: # First character of new word
|
|
# Convert from milliseconds to seconds and add cumulative offset
|
|
word_start_time = cumulative_time + (char_start_times_ms[i] / 1000.0)
|
|
current_word += char
|
|
|
|
# Build result for complete words
|
|
word_times = list(zip(words, word_start_times))
|
|
|
|
# Return any incomplete word at the end of this chunk
|
|
new_partial_word = current_word if current_word else ""
|
|
new_partial_word_start_time = word_start_time if word_start_time is not None else 0.0
|
|
|
|
return (word_times, new_partial_word, new_partial_word_start_time)
|
|
|
|
|
|
class ElevenLabsTTSService(AudioContextWordTTSService):
|
|
"""ElevenLabs WebSocket-based TTS service with word timestamps.
|
|
|
|
Provides real-time text-to-speech using ElevenLabs' WebSocket streaming API.
|
|
Supports word-level timestamps, audio context management, and various voice
|
|
customization options including stability, similarity boost, and speed controls.
|
|
"""
|
|
|
|
class InputParams(BaseModel):
|
|
"""Input parameters for ElevenLabs TTS configuration.
|
|
|
|
Parameters:
|
|
language: Language to use for synthesis.
|
|
stability: Voice stability control (0.0 to 1.0).
|
|
similarity_boost: Similarity boost control (0.0 to 1.0).
|
|
style: Style control for voice expression (0.0 to 1.0).
|
|
use_speaker_boost: Whether to use speaker boost enhancement.
|
|
speed: Voice speed control (0.7 to 1.2).
|
|
auto_mode: Whether to enable automatic mode optimization.
|
|
enable_ssml_parsing: Whether to parse SSML tags in text.
|
|
enable_logging: Whether to enable ElevenLabs logging.
|
|
apply_text_normalization: Text normalization mode ("auto", "on", "off").
|
|
pronunciation_dictionary_locators: List of pronunciation dictionary locators to use.
|
|
"""
|
|
|
|
language: Optional[Language] = None
|
|
stability: Optional[float] = None
|
|
similarity_boost: Optional[float] = None
|
|
style: Optional[float] = None
|
|
use_speaker_boost: Optional[bool] = None
|
|
speed: Optional[float] = None
|
|
auto_mode: Optional[bool] = True
|
|
enable_ssml_parsing: Optional[bool] = None
|
|
enable_logging: Optional[bool] = None
|
|
apply_text_normalization: Optional[Literal["auto", "on", "off"]] = None
|
|
pronunciation_dictionary_locators: Optional[List[PronunciationDictionaryLocator]] = None
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
api_key: str,
|
|
voice_id: str,
|
|
model: str = "eleven_turbo_v2_5",
|
|
url: str = "wss://api.elevenlabs.io",
|
|
sample_rate: Optional[int] = None,
|
|
params: Optional[InputParams] = None,
|
|
aggregate_sentences: Optional[bool] = True,
|
|
**kwargs,
|
|
):
|
|
"""Initialize the ElevenLabs TTS service.
|
|
|
|
Args:
|
|
api_key: ElevenLabs API key for authentication.
|
|
voice_id: ID of the voice to use for synthesis.
|
|
model: TTS model to use (e.g., "eleven_turbo_v2_5").
|
|
url: WebSocket URL for ElevenLabs TTS API.
|
|
sample_rate: Audio sample rate. If None, uses default.
|
|
params: Additional input parameters for voice customization.
|
|
aggregate_sentences: Whether to aggregate sentences within the TTSService.
|
|
**kwargs: Additional arguments passed to the parent service.
|
|
"""
|
|
# Aggregating sentences still gives cleaner-sounding results and fewer
|
|
# artifacts than streaming one word at a time. On average, waiting for a
|
|
# full sentence should only "cost" us 15ms or so with GPT-4o or a Llama
|
|
# 3 model, and it's worth it for the better audio quality.
|
|
#
|
|
# We also don't want to automatically push LLM response text frames,
|
|
# because the context aggregators will add them to the LLM context even
|
|
# if we're interrupted. ElevenLabs gives us word-by-word timestamps. We
|
|
# can use those to generate text frames ourselves aligned with the
|
|
# playout timing of the audio!
|
|
#
|
|
# Finally, ElevenLabs doesn't provide information on when the bot stops
|
|
# speaking for a while, so we want the parent class to send TTSStopFrame
|
|
# after a short period not receiving any audio.
|
|
super().__init__(
|
|
aggregate_sentences=aggregate_sentences,
|
|
push_text_frames=False,
|
|
push_stop_frames=True,
|
|
pause_frame_processing=True,
|
|
sample_rate=sample_rate,
|
|
**kwargs,
|
|
)
|
|
|
|
params = params or ElevenLabsTTSService.InputParams()
|
|
|
|
self._api_key = api_key
|
|
self._url = url
|
|
self._settings = {
|
|
"language": self.language_to_service_language(params.language)
|
|
if params.language
|
|
else None,
|
|
"stability": params.stability,
|
|
"similarity_boost": params.similarity_boost,
|
|
"style": params.style,
|
|
"use_speaker_boost": params.use_speaker_boost,
|
|
"speed": params.speed,
|
|
"auto_mode": str(params.auto_mode).lower(),
|
|
"enable_ssml_parsing": params.enable_ssml_parsing,
|
|
"enable_logging": params.enable_logging,
|
|
"apply_text_normalization": params.apply_text_normalization,
|
|
}
|
|
self.set_model_name(model)
|
|
self.set_voice(voice_id)
|
|
self._output_format = "" # initialized in start()
|
|
self._voice_settings = self._set_voice_settings()
|
|
self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators
|
|
|
|
# Indicates if we have sent TTSStartedFrame. It will reset to False when
|
|
# there's an interruption or TTSStoppedFrame.
|
|
self._started = False
|
|
self._cumulative_time = 0
|
|
# Track partial words that span across alignment chunks
|
|
self._partial_word = ""
|
|
self._partial_word_start_time = 0.0
|
|
|
|
# Context management for v1 multi API
|
|
self._context_id = None
|
|
self._receive_task = None
|
|
self._keepalive_task = None
|
|
|
|
def can_generate_metrics(self) -> bool:
|
|
"""Check if this service can generate processing metrics.
|
|
|
|
Returns:
|
|
True, as ElevenLabs service supports metrics generation.
|
|
"""
|
|
return True
|
|
|
|
def language_to_service_language(self, language: Language) -> Optional[str]:
|
|
"""Convert a Language enum to ElevenLabs language format.
|
|
|
|
Args:
|
|
language: The language to convert.
|
|
|
|
Returns:
|
|
The ElevenLabs-specific language code, or None if not supported.
|
|
"""
|
|
return language_to_elevenlabs_language(language)
|
|
|
|
def _set_voice_settings(self):
|
|
return build_elevenlabs_voice_settings(self._settings)
|
|
|
|
async def set_model(self, model: str):
|
|
"""Set the TTS model and reconnect.
|
|
|
|
Args:
|
|
model: The model name to use for synthesis.
|
|
"""
|
|
await super().set_model(model)
|
|
logger.info(f"Switching TTS model to: [{model}]")
|
|
await self._disconnect()
|
|
await self._connect()
|
|
|
|
async def _update_settings(self, settings: Mapping[str, Any]):
|
|
"""Update service settings and reconnect if voice, model, or language changed."""
|
|
# Track previous values for settings that require reconnection
|
|
prev_voice = self._voice_id
|
|
prev_model = self.model_name
|
|
prev_language = self._settings.get("language")
|
|
# Create snapshot of current voice settings to detect changes after update
|
|
prev_voice_settings = self._voice_settings.copy() if self._voice_settings else None
|
|
|
|
await super()._update_settings(settings)
|
|
|
|
# Update voice settings for the next context creation
|
|
self._voice_settings = self._set_voice_settings()
|
|
|
|
# Check if URL-level settings changed (these require reconnection)
|
|
url_changed = (
|
|
prev_voice != self._voice_id
|
|
or prev_model != self.model_name
|
|
or prev_language != self._settings.get("language")
|
|
)
|
|
|
|
# Check if only voice settings changed (speed, stability, etc.)
|
|
voice_settings_changed = prev_voice_settings != self._voice_settings
|
|
|
|
if url_changed:
|
|
# These settings are in the WebSocket URL, so we need to reconnect
|
|
logger.debug(
|
|
f"URL-level setting changed (voice/model/language), reconnecting WebSocket"
|
|
)
|
|
await self._disconnect()
|
|
await self._connect()
|
|
elif voice_settings_changed and self._context_id:
|
|
# Voice settings can be updated by closing current context
|
|
# so new one gets created with updated voice settings
|
|
logger.debug(f"Voice settings changed, closing current context to apply changes")
|
|
try:
|
|
if self._websocket:
|
|
await self._websocket.send(
|
|
json.dumps({"context_id": self._context_id, "close_context": True})
|
|
)
|
|
except Exception as e:
|
|
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
|
|
self._context_id = None
|
|
self._started = False
|
|
|
|
async def start(self, frame: StartFrame):
|
|
"""Start the ElevenLabs TTS service.
|
|
|
|
Args:
|
|
frame: The start frame containing initialization parameters.
|
|
"""
|
|
await super().start(frame)
|
|
self._output_format = output_format_from_sample_rate(self.sample_rate)
|
|
await self._connect()
|
|
|
|
async def stop(self, frame: EndFrame):
|
|
"""Stop the ElevenLabs TTS service.
|
|
|
|
Args:
|
|
frame: The end frame.
|
|
"""
|
|
await super().stop(frame)
|
|
await self._disconnect()
|
|
|
|
async def cancel(self, frame: CancelFrame):
|
|
"""Cancel the ElevenLabs TTS service.
|
|
|
|
Args:
|
|
frame: The cancel frame.
|
|
"""
|
|
await super().cancel(frame)
|
|
await self._disconnect()
|
|
|
|
async def flush_audio(self):
|
|
"""Flush any pending audio and finalize the current context."""
|
|
if not self._context_id or not self._websocket:
|
|
return
|
|
logger.trace(f"{self}: flushing audio")
|
|
msg = {"context_id": self._context_id, "flush": True}
|
|
await self._websocket.send(json.dumps(msg))
|
|
|
|
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
|
|
"""Push a frame and handle state changes.
|
|
|
|
Args:
|
|
frame: The frame to push.
|
|
direction: The direction to push the frame.
|
|
"""
|
|
await super().push_frame(frame, direction)
|
|
if isinstance(frame, (TTSStoppedFrame, InterruptionFrame)):
|
|
self._started = False
|
|
if isinstance(frame, TTSStoppedFrame):
|
|
await self.add_word_timestamps([("Reset", 0)])
|
|
|
|
async def _connect(self):
|
|
await super()._connect()
|
|
|
|
await self._connect_websocket()
|
|
|
|
if self._websocket and not self._receive_task:
|
|
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
|
|
|
|
if self._websocket and not self._keepalive_task:
|
|
self._keepalive_task = self.create_task(self._keepalive_task_handler())
|
|
|
|
async def _disconnect(self):
|
|
await super()._disconnect()
|
|
|
|
if self._receive_task:
|
|
await self.cancel_task(self._receive_task)
|
|
self._receive_task = None
|
|
|
|
if self._keepalive_task:
|
|
await self.cancel_task(self._keepalive_task)
|
|
self._keepalive_task = None
|
|
|
|
await self._disconnect_websocket()
|
|
|
|
async def _connect_websocket(self):
|
|
try:
|
|
if self._websocket and self._websocket.state is State.OPEN:
|
|
return
|
|
|
|
logger.debug("Connecting to ElevenLabs")
|
|
|
|
voice_id = self._voice_id
|
|
model = self.model_name
|
|
output_format = self._output_format
|
|
url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}"
|
|
|
|
if self._settings["enable_ssml_parsing"]:
|
|
url += f"&enable_ssml_parsing={self._settings['enable_ssml_parsing']}"
|
|
|
|
if self._settings["enable_logging"]:
|
|
url += f"&enable_logging={self._settings['enable_logging']}"
|
|
|
|
if self._settings["apply_text_normalization"] is not None:
|
|
url += f"&apply_text_normalization={self._settings['apply_text_normalization']}"
|
|
|
|
# Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS
|
|
language = self._settings["language"]
|
|
if model in ELEVENLABS_MULTILINGUAL_MODELS and language is not None:
|
|
url += f"&language_code={language}"
|
|
logger.debug(f"Using language code: {language}")
|
|
elif language is not None:
|
|
logger.warning(
|
|
f"Language code [{language}] not applied. Language codes can only be used with multilingual models: {', '.join(sorted(ELEVENLABS_MULTILINGUAL_MODELS))}"
|
|
)
|
|
|
|
# Set max websocket message size to 16MB for large audio responses
|
|
self._websocket = await websocket_connect(
|
|
url, max_size=16 * 1024 * 1024, additional_headers={"xi-api-key": self._api_key}
|
|
)
|
|
|
|
await self._call_event_handler("on_connected")
|
|
except Exception as e:
|
|
self._websocket = None
|
|
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
|
|
await self._call_event_handler("on_connection_error", f"{e}")
|
|
|
|
async def _disconnect_websocket(self):
|
|
try:
|
|
await self.stop_all_metrics()
|
|
|
|
if self._websocket:
|
|
logger.debug("Disconnecting from ElevenLabs")
|
|
# Close all contexts and the socket
|
|
if self._context_id:
|
|
await self._websocket.send(json.dumps({"close_socket": True}))
|
|
await self._websocket.close()
|
|
logger.debug("Disconnected from ElevenLabs")
|
|
except Exception as e:
|
|
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
|
|
finally:
|
|
self._started = False
|
|
self._context_id = None
|
|
self._websocket = None
|
|
await self._call_event_handler("on_disconnected")
|
|
|
|
def _get_websocket(self):
|
|
if self._websocket:
|
|
return self._websocket
|
|
raise Exception("Websocket not connected")
|
|
|
|
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
|
|
"""Handle interruption by closing the current context."""
|
|
await super()._handle_interruption(frame, direction)
|
|
|
|
# Close the current context when interrupted without closing the websocket
|
|
if self._context_id and self._websocket:
|
|
logger.trace(f"Closing context {self._context_id} due to interruption")
|
|
try:
|
|
# ElevenLabs requires that Pipecat manages the contexts and closes them
|
|
# when they're not longer in use. Since an InterruptionFrame is pushed
|
|
# every time the user speaks, we'll use this as a trigger to close the context
|
|
# and reset the state.
|
|
# Note: We do not need to call remove_audio_context here, as the context is
|
|
# automatically reset when super ()._handle_interruption is called.
|
|
await self._websocket.send(
|
|
json.dumps({"context_id": self._context_id, "close_context": True})
|
|
)
|
|
except Exception as e:
|
|
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
|
|
self._context_id = None
|
|
self._started = False
|
|
self._partial_word = ""
|
|
self._partial_word_start_time = 0.0
|
|
|
|
async def _receive_messages(self):
|
|
"""Handle incoming WebSocket messages from ElevenLabs."""
|
|
async for message in self._get_websocket():
|
|
msg = json.loads(message)
|
|
|
|
received_ctx_id = msg.get("contextId")
|
|
|
|
# Handle final messages first, regardless of context availability
|
|
# At the moment, this message is received AFTER the close_context message is
|
|
# sent, so it doesn't serve any functional purpose. For now, we'll just log it.
|
|
if msg.get("isFinal") is True:
|
|
logger.trace(f"Received final message for context {received_ctx_id}")
|
|
continue
|
|
|
|
# Check if this message belongs to the current context.
|
|
if not self.audio_context_available(received_ctx_id):
|
|
if self._context_id == received_ctx_id:
|
|
logger.debug(
|
|
f"Received a delayed message, recreating the context: {self._context_id}"
|
|
)
|
|
await self.create_audio_context(self._context_id)
|
|
else:
|
|
# This can happen if a message is received _after_ we have closed a context
|
|
# due to user interruption but _before_ the `isFinal` message for the context
|
|
# is received.
|
|
logger.debug(f"Ignoring message from unavailable context: {received_ctx_id}")
|
|
continue
|
|
|
|
if msg.get("audio"):
|
|
await self.stop_ttfb_metrics()
|
|
await self.start_word_timestamps()
|
|
|
|
audio = base64.b64decode(msg["audio"])
|
|
frame = TTSAudioRawFrame(audio, self.sample_rate, 1)
|
|
await self.append_to_audio_context(received_ctx_id, frame)
|
|
|
|
if msg.get("alignment"):
|
|
alignment = msg["alignment"]
|
|
word_times, self._partial_word, self._partial_word_start_time = (
|
|
calculate_word_times(
|
|
alignment,
|
|
self._cumulative_time,
|
|
self._partial_word,
|
|
self._partial_word_start_time,
|
|
)
|
|
)
|
|
|
|
if word_times:
|
|
await self.add_word_timestamps(word_times)
|
|
|
|
# Calculate the actual end time of this audio chunk
|
|
char_start_times_ms = alignment.get("charStartTimesMs", [])
|
|
char_durations_ms = alignment.get("charDurationsMs", [])
|
|
|
|
if char_start_times_ms and char_durations_ms:
|
|
# End time = start time of last character + duration of last character
|
|
chunk_end_time_ms = char_start_times_ms[-1] + char_durations_ms[-1]
|
|
chunk_end_time_seconds = chunk_end_time_ms / 1000.0
|
|
self._cumulative_time += chunk_end_time_seconds
|
|
else:
|
|
# Fallback: use the last word's start time (current behavior)
|
|
self._cumulative_time = word_times[-1][1]
|
|
logger.warning(
|
|
"_receive_messages: using fallback timing method - consider investigating alignment data structure"
|
|
)
|
|
|
|
async def _keepalive_task_handler(self):
|
|
"""Send periodic keepalive messages to maintain WebSocket connection."""
|
|
KEEPALIVE_SLEEP = 10
|
|
while True:
|
|
await asyncio.sleep(KEEPALIVE_SLEEP)
|
|
try:
|
|
if self._websocket and self._websocket.state is State.OPEN:
|
|
if self._context_id:
|
|
# Send keepalive with context ID to keep the connection alive
|
|
keepalive_message = {
|
|
"text": "",
|
|
"context_id": self._context_id,
|
|
}
|
|
logger.trace(f"Sending keepalive for context {self._context_id}")
|
|
else:
|
|
# It's possible to have a user interruption which clears the context
|
|
# without generating a new TTS response. In this case, we'll just send
|
|
# an empty message to keep the connection alive.
|
|
keepalive_message = {"text": ""}
|
|
logger.trace("Sending keepalive without context")
|
|
await self._websocket.send(json.dumps(keepalive_message))
|
|
except websockets.ConnectionClosed as e:
|
|
logger.warning(f"{self} keepalive error: {e}")
|
|
break
|
|
|
|
async def _send_text(self, text: str):
|
|
"""Send text to the WebSocket for synthesis."""
|
|
if self._websocket and self._context_id:
|
|
msg = {"text": text, "context_id": self._context_id}
|
|
await self._websocket.send(json.dumps(msg))
|
|
|
|
@traced_tts
|
|
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
|
|
"""Generate speech from text using ElevenLabs' streaming WebSocket API.
|
|
|
|
Args:
|
|
text: The text to synthesize into speech.
|
|
|
|
Yields:
|
|
Frame: Audio frames containing the synthesized speech.
|
|
"""
|
|
logger.debug(f"{self}: Generating TTS [{text}]")
|
|
|
|
try:
|
|
if not self._websocket or self._websocket.state is State.CLOSED:
|
|
await self._connect()
|
|
|
|
try:
|
|
if not self._started:
|
|
await self.start_ttfb_metrics()
|
|
yield TTSStartedFrame()
|
|
self._started = True
|
|
self._cumulative_time = 0
|
|
self._partial_word = ""
|
|
self._partial_word_start_time = 0.0
|
|
# If a context ID does not exist, create a new one and
|
|
# register it. If an ID exists, that means the Pipeline
|
|
# doesn't allow user interruptions, so continue using the
|
|
# current ID. When interruptions are allowed, user speech
|
|
# results in an interruption, which resets the context ID.
|
|
if not self._context_id:
|
|
self._context_id = str(uuid.uuid4())
|
|
if not self.audio_context_available(self._context_id):
|
|
await self.create_audio_context(self._context_id)
|
|
|
|
# Initialize context with voice settings and pronunciation dictionaries
|
|
msg = {"text": " ", "context_id": self._context_id}
|
|
if self._voice_settings:
|
|
msg["voice_settings"] = self._voice_settings
|
|
if self._pronunciation_dictionary_locators:
|
|
msg["pronunciation_dictionary_locators"] = [
|
|
locator.model_dump()
|
|
for locator in self._pronunciation_dictionary_locators
|
|
]
|
|
await self._websocket.send(json.dumps(msg))
|
|
logger.trace(f"Created new context {self._context_id}")
|
|
|
|
await self._send_text(text)
|
|
await self.start_tts_usage_metrics(text)
|
|
except Exception as e:
|
|
yield TTSStoppedFrame()
|
|
yield ErrorFrame(error=f"Unknown error occurred: {e}")
|
|
self._started = False
|
|
return
|
|
yield None
|
|
except Exception as e:
|
|
yield ErrorFrame(error=f"Unknown error occurred: {e}")
|
|
|
|
|
|
class ElevenLabsHttpTTSService(WordTTSService):
|
|
"""ElevenLabs HTTP-based TTS service with word timestamps.
|
|
|
|
Provides text-to-speech using ElevenLabs' HTTP streaming API for simpler,
|
|
non-WebSocket integration. Suitable for use cases where streaming WebSocket
|
|
connection is not required or desired.
|
|
"""
|
|
|
|
class InputParams(BaseModel):
|
|
"""Input parameters for ElevenLabs HTTP TTS configuration.
|
|
|
|
Parameters:
|
|
language: Language to use for synthesis.
|
|
optimize_streaming_latency: Latency optimization level (0-4).
|
|
stability: Voice stability control (0.0 to 1.0).
|
|
similarity_boost: Similarity boost control (0.0 to 1.0).
|
|
style: Style control for voice expression (0.0 to 1.0).
|
|
use_speaker_boost: Whether to use speaker boost enhancement.
|
|
speed: Voice speed control (0.25 to 4.0).
|
|
apply_text_normalization: Text normalization mode ("auto", "on", "off").
|
|
pronunciation_dictionary_locators: List of pronunciation dictionary locators to use.
|
|
"""
|
|
|
|
language: Optional[Language] = None
|
|
optimize_streaming_latency: Optional[int] = None
|
|
stability: Optional[float] = None
|
|
similarity_boost: Optional[float] = None
|
|
style: Optional[float] = None
|
|
use_speaker_boost: Optional[bool] = None
|
|
speed: Optional[float] = None
|
|
apply_text_normalization: Optional[Literal["auto", "on", "off"]] = None
|
|
pronunciation_dictionary_locators: Optional[List[PronunciationDictionaryLocator]] = None
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
api_key: str,
|
|
voice_id: str,
|
|
aiohttp_session: aiohttp.ClientSession,
|
|
model: str = "eleven_turbo_v2_5",
|
|
base_url: str = "https://api.elevenlabs.io",
|
|
sample_rate: Optional[int] = None,
|
|
params: Optional[InputParams] = None,
|
|
aggregate_sentences: Optional[bool] = True,
|
|
**kwargs,
|
|
):
|
|
"""Initialize the ElevenLabs HTTP TTS service.
|
|
|
|
Args:
|
|
api_key: ElevenLabs API key for authentication.
|
|
voice_id: ID of the voice to use for synthesis.
|
|
aiohttp_session: aiohttp ClientSession for HTTP requests.
|
|
model: TTS model to use (e.g., "eleven_turbo_v2_5").
|
|
base_url: Base URL for ElevenLabs HTTP API.
|
|
sample_rate: Audio sample rate. If None, uses default.
|
|
params: Additional input parameters for voice customization.
|
|
aggregate_sentences: Whether to aggregate sentences within the TTSService.
|
|
**kwargs: Additional arguments passed to the parent service.
|
|
"""
|
|
super().__init__(
|
|
aggregate_sentences=aggregate_sentences,
|
|
push_text_frames=False,
|
|
push_stop_frames=True,
|
|
sample_rate=sample_rate,
|
|
**kwargs,
|
|
)
|
|
|
|
params = params or ElevenLabsHttpTTSService.InputParams()
|
|
|
|
self._api_key = api_key
|
|
self._base_url = base_url
|
|
self._params = params
|
|
self._session = aiohttp_session
|
|
|
|
self._settings = {
|
|
"language": self.language_to_service_language(params.language)
|
|
if params.language
|
|
else None,
|
|
"optimize_streaming_latency": params.optimize_streaming_latency,
|
|
"stability": params.stability,
|
|
"similarity_boost": params.similarity_boost,
|
|
"style": params.style,
|
|
"use_speaker_boost": params.use_speaker_boost,
|
|
"speed": params.speed,
|
|
"apply_text_normalization": params.apply_text_normalization,
|
|
}
|
|
self.set_model_name(model)
|
|
self.set_voice(voice_id)
|
|
self._output_format = "" # initialized in start()
|
|
self._voice_settings = self._set_voice_settings()
|
|
self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators
|
|
|
|
# Track cumulative time to properly sequence word timestamps across utterances
|
|
self._cumulative_time = 0
|
|
self._started = False
|
|
|
|
# Store previous text for context within a turn
|
|
self._previous_text = ""
|
|
|
|
# Track partial words that span across alignment chunks
|
|
self._partial_word = ""
|
|
self._partial_word_start_time = 0.0
|
|
|
|
def language_to_service_language(self, language: Language) -> Optional[str]:
|
|
"""Convert pipecat Language to ElevenLabs language code.
|
|
|
|
Args:
|
|
language: The language to convert.
|
|
|
|
Returns:
|
|
The ElevenLabs-specific language code, or None if not supported.
|
|
"""
|
|
return language_to_elevenlabs_language(language)
|
|
|
|
def can_generate_metrics(self) -> bool:
|
|
"""Check if this service can generate processing metrics.
|
|
|
|
Returns:
|
|
True, as ElevenLabs HTTP service supports metrics generation.
|
|
"""
|
|
return True
|
|
|
|
def _set_voice_settings(self):
|
|
return build_elevenlabs_voice_settings(self._settings)
|
|
|
|
async def _update_settings(self, settings: Mapping[str, Any]):
|
|
await super()._update_settings(settings)
|
|
# Update voice settings for the next context creation
|
|
self._voice_settings = self._set_voice_settings()
|
|
|
|
def _reset_state(self):
|
|
"""Reset internal state variables."""
|
|
self._cumulative_time = 0
|
|
self._started = False
|
|
self._previous_text = ""
|
|
self._partial_word = ""
|
|
self._partial_word_start_time = 0.0
|
|
logger.debug(f"{self}: Reset internal state")
|
|
|
|
async def start(self, frame: StartFrame):
|
|
"""Start the ElevenLabs HTTP TTS service.
|
|
|
|
Args:
|
|
frame: The start frame containing initialization parameters.
|
|
"""
|
|
await super().start(frame)
|
|
self._output_format = output_format_from_sample_rate(self.sample_rate)
|
|
self._reset_state()
|
|
|
|
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
|
|
"""Push a frame and handle state changes.
|
|
|
|
Args:
|
|
frame: The frame to push.
|
|
direction: The direction to push the frame.
|
|
"""
|
|
await super().push_frame(frame, direction)
|
|
if isinstance(frame, (InterruptionFrame, TTSStoppedFrame)):
|
|
# Reset timing on interruption or stop
|
|
self._reset_state()
|
|
|
|
if isinstance(frame, TTSStoppedFrame):
|
|
await self.add_word_timestamps([("Reset", 0)])
|
|
|
|
elif isinstance(frame, LLMFullResponseEndFrame):
|
|
# End of turn - reset previous text
|
|
self._previous_text = ""
|
|
|
|
def calculate_word_times(self, alignment_info: Mapping[str, Any]) -> List[Tuple[str, float]]:
|
|
"""Calculate word timing from character alignment data.
|
|
|
|
This method handles partial words that may span across multiple alignment chunks.
|
|
|
|
Args:
|
|
alignment_info: Character timing data from ElevenLabs.
|
|
|
|
Returns:
|
|
List of (word, timestamp) pairs for complete words in this chunk.
|
|
|
|
Example input data::
|
|
|
|
{
|
|
"characters": [" ", "H", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d"],
|
|
"character_start_times_seconds": [0.0, 0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
|
|
"character_end_times_seconds": [0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
|
|
}
|
|
|
|
Would produce word times (with cumulative_time=0)::
|
|
|
|
[("Hello", 0.1), ("world", 0.5)]
|
|
"""
|
|
chars = alignment_info.get("characters", [])
|
|
char_start_times = alignment_info.get("character_start_times_seconds", [])
|
|
|
|
if not chars or not char_start_times or len(chars) != len(char_start_times):
|
|
logger.warning(
|
|
f"Invalid alignment data: chars={len(chars)}, times={len(char_start_times)}"
|
|
)
|
|
return []
|
|
|
|
# Build the words and find their start times
|
|
words = []
|
|
word_start_times = []
|
|
# Start with any partial word from previous chunk
|
|
current_word = self._partial_word
|
|
word_start_time = self._partial_word_start_time if self._partial_word else None
|
|
|
|
for i, char in enumerate(chars):
|
|
if char == " ":
|
|
if current_word: # Only add non-empty words
|
|
words.append(current_word)
|
|
word_start_times.append(word_start_time)
|
|
current_word = ""
|
|
word_start_time = None
|
|
else:
|
|
if word_start_time is None: # First character of a new word
|
|
# Use time of the first character of the word, offset by cumulative time
|
|
word_start_time = self._cumulative_time + char_start_times[i]
|
|
current_word += char
|
|
|
|
# Store any incomplete word at the end of this chunk
|
|
self._partial_word = current_word if current_word else ""
|
|
self._partial_word_start_time = word_start_time if word_start_time is not None else 0.0
|
|
|
|
# Create word-time pairs for complete words only
|
|
word_times = list(zip(words, word_start_times))
|
|
|
|
return word_times
|
|
|
|
@traced_tts
|
|
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
|
|
"""Generate speech from text using ElevenLabs streaming API with timestamps.
|
|
|
|
Makes a request to the ElevenLabs API to generate audio and timing data.
|
|
Tracks the duration of each utterance to ensure correct sequencing.
|
|
Includes previous text as context for better prosody continuity.
|
|
|
|
Args:
|
|
text: Text to convert to speech.
|
|
|
|
Yields:
|
|
Frame: Audio and control frames containing the synthesized speech.
|
|
"""
|
|
logger.debug(f"{self}: Generating TTS [{text}]")
|
|
|
|
# Use the with-timestamps endpoint
|
|
url = f"{self._base_url}/v1/text-to-speech/{self._voice_id}/stream/with-timestamps"
|
|
|
|
payload: Dict[str, Union[str, Dict[str, Union[float, bool]]]] = {
|
|
"text": text,
|
|
"model_id": self._model_name,
|
|
}
|
|
|
|
# Include previous text as context if available
|
|
if self._previous_text:
|
|
payload["previous_text"] = self._previous_text
|
|
|
|
if self._voice_settings:
|
|
payload["voice_settings"] = self._voice_settings
|
|
|
|
if self._pronunciation_dictionary_locators:
|
|
payload["pronunciation_dictionary_locators"] = [
|
|
locator.model_dump() for locator in self._pronunciation_dictionary_locators
|
|
]
|
|
|
|
if self._settings["apply_text_normalization"] is not None:
|
|
payload["apply_text_normalization"] = self._settings["apply_text_normalization"]
|
|
|
|
language = self._settings["language"]
|
|
if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language:
|
|
payload["language_code"] = language
|
|
logger.debug(f"Using language code: {language}")
|
|
elif language:
|
|
logger.warning(
|
|
f"Language code [{language}] not applied. Language codes can only be used with multilingual models: {', '.join(sorted(ELEVENLABS_MULTILINGUAL_MODELS))}"
|
|
)
|
|
|
|
headers = {
|
|
"xi-api-key": self._api_key,
|
|
"Content-Type": "application/json",
|
|
}
|
|
|
|
# Build query parameters
|
|
params = {
|
|
"output_format": self._output_format,
|
|
}
|
|
if self._settings["optimize_streaming_latency"] is not None:
|
|
params["optimize_streaming_latency"] = self._settings["optimize_streaming_latency"]
|
|
|
|
try:
|
|
await self.start_ttfb_metrics()
|
|
|
|
async with self._session.post(
|
|
url, json=payload, headers=headers, params=params
|
|
) as response:
|
|
if response.status != 200:
|
|
error_text = await response.text()
|
|
yield ErrorFrame(error=f"ElevenLabs API error: {error_text}")
|
|
return
|
|
|
|
await self.start_tts_usage_metrics(text)
|
|
|
|
# Start TTS sequence if not already started
|
|
if not self._started:
|
|
await self.start_word_timestamps()
|
|
yield TTSStartedFrame()
|
|
self._started = True
|
|
|
|
# Track the duration of this utterance based on the last character's end time
|
|
utterance_duration = 0
|
|
async for line in response.content:
|
|
line_str = line.decode("utf-8").strip()
|
|
if not line_str:
|
|
continue
|
|
|
|
try:
|
|
# Parse the JSON object
|
|
data = json.loads(line_str)
|
|
|
|
# Process audio if present
|
|
if data and "audio_base64" in data:
|
|
await self.stop_ttfb_metrics()
|
|
audio = base64.b64decode(data["audio_base64"])
|
|
yield TTSAudioRawFrame(audio, self.sample_rate, 1)
|
|
|
|
# Process alignment if present
|
|
if data and "alignment" in data:
|
|
alignment = data["alignment"]
|
|
if alignment: # Ensure alignment is not None
|
|
# Get end time of the last character in this chunk
|
|
char_end_times = alignment.get("character_end_times_seconds", [])
|
|
if char_end_times:
|
|
chunk_end_time = char_end_times[-1]
|
|
# Update to the longest end time seen so far
|
|
utterance_duration = max(utterance_duration, chunk_end_time)
|
|
|
|
# Calculate word timestamps
|
|
word_times = self.calculate_word_times(alignment)
|
|
if word_times:
|
|
await self.add_word_timestamps(word_times)
|
|
except json.JSONDecodeError as e:
|
|
logger.warning(f"Failed to parse JSON from stream: {e}")
|
|
continue
|
|
except Exception as e:
|
|
yield ErrorFrame(error=f"Unknown error occurred: {e}")
|
|
continue
|
|
|
|
# After processing all chunks, emit any remaining partial word
|
|
# since this is the end of the utterance
|
|
if self._partial_word:
|
|
final_word_time = [(self._partial_word, self._partial_word_start_time)]
|
|
await self.add_word_timestamps(final_word_time)
|
|
self._partial_word = ""
|
|
self._partial_word_start_time = 0.0
|
|
|
|
# After processing all chunks, add the total utterance duration
|
|
# to the cumulative time to ensure next utterance starts after this one
|
|
if utterance_duration > 0:
|
|
self._cumulative_time += utterance_duration
|
|
|
|
# Append the current text to previous_text for context continuity
|
|
# Only add a space if there's already text
|
|
if self._previous_text:
|
|
self._previous_text += " " + text
|
|
else:
|
|
self._previous_text = text
|
|
|
|
except Exception as e:
|
|
yield ErrorFrame(error=f"Unknown error occurred: {e}")
|
|
finally:
|
|
await self.stop_ttfb_metrics()
|
|
# Let the parent class handle TTSStoppedFrame
|