From 4a23e138b14ecdfe497135a6045e8e8031953922 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 15 Apr 2025 21:29:19 -0400 Subject: [PATCH 1/3] Added word/timestamp pairs to ElevenLabsHttpTTSService --- src/pipecat/services/elevenlabs/tts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index d77e4199c..832ed7309 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -12,6 +12,7 @@ from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, import aiohttp from loguru import logger from pydantic import BaseModel, model_validator +from sentry_sdk import push_scope from pipecat.frames.frames import ( CancelFrame, From 1e0a9d7b06164c22b50a30f59fd26d5d33cd3d11 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 15 Apr 2025 22:39:36 -0400 Subject: [PATCH 2/3] Add previous_text context to ElevenLabsHttpTTSService --- CHANGELOG.md | 3 +++ src/pipecat/services/elevenlabs/tts.py | 28 +++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3333c27d..1324e2dbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 you to control aggregator settings. You can now pass these arguments when creating aggregator pairs with `create_context_aggregator()`. +- Added `previous_text` context support to ElevenLabsHttpTTSService, improving + speech consistency across sentences within an LLM response. + - Added word/timestamp pairs to `ElevenLabsHttpTTSService`. - It is now possible to disable `SoundfileMixer` when created. You can then use diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 832ed7309..b3e203b35 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -12,13 +12,13 @@ from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, import aiohttp from loguru import logger from pydantic import BaseModel, model_validator -from sentry_sdk import push_scope from pipecat.frames.frames import ( CancelFrame, EndFrame, ErrorFrame, Frame, + LLMFullResponseEndFrame, StartFrame, StartInterruptionFrame, TTSAudioRawFrame, @@ -509,6 +509,9 @@ class ElevenLabsHttpTTSService(WordTTSService): self._cumulative_time = 0 self._started = False + # Store previous text for context within a turn + self._previous_text = "" + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert pipecat Language to ElevenLabs language code.""" return language_to_elevenlabs_language(language) @@ -526,6 +529,7 @@ class ElevenLabsHttpTTSService(WordTTSService): self._output_format = output_format_from_sample_rate(self.sample_rate) self._cumulative_time = 0 self._started = False + self._previous_text = "" async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): await super().push_frame(frame, direction) @@ -533,9 +537,15 @@ class ElevenLabsHttpTTSService(WordTTSService): # Reset timing on interruption or stop self._started = False self._cumulative_time = 0 + self._previous_text = "" + if isinstance(frame, TTSStoppedFrame): await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)]) + elif isinstance(frame, LLMFullResponseEndFrame): + # End of turn - reset previous text + self._previous_text = "" + def calculate_word_times(self, alignment_info: Mapping[str, Any]) -> List[Tuple[str, float]]: """Calculate word timing from character alignment data. @@ -598,6 +608,10 @@ class ElevenLabsHttpTTSService(WordTTSService): async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using ElevenLabs streaming API with timestamps. + Makes a request to the ElevenLabs API to generate audio and timing data. + Tracks the duration of each utterance to ensure correct sequencing. + Includes previous text as context for better prosody continuity. + Args: text: Text to convert to speech @@ -614,6 +628,11 @@ class ElevenLabsHttpTTSService(WordTTSService): "model_id": self._model_name, } + # Include previous text as context if available + if self._previous_text: + payload["previous_text"] = self._previous_text + print(f"Previous text: {self._previous_text}") + if self._voice_settings: payload["voice_settings"] = self._voice_settings @@ -702,6 +721,13 @@ class ElevenLabsHttpTTSService(WordTTSService): if utterance_duration > 0: self._cumulative_time += utterance_duration + # Append the current text to previous_text for context continuity + # Only add a space if there's already text + if self._previous_text: + self._previous_text += " " + text + else: + self._previous_text = text + except Exception as e: logger.error(f"Error in run_tts: {e}") yield ErrorFrame(error=str(e)) From 2dafbee2aa26b9c7062168a8e4f8a597f010c1cf Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 16 Apr 2025 22:29:33 -0400 Subject: [PATCH 3/3] Code review fixes --- src/pipecat/services/elevenlabs/tts.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index b3e203b35..acb4a7b12 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -523,21 +523,24 @@ class ElevenLabsHttpTTSService(WordTTSService): def _set_voice_settings(self): return build_elevenlabs_voice_settings(self._settings) + def _reset_state(self): + """Reset internal state variables.""" + self._cumulative_time = 0 + self._started = False + self._previous_text = "" + logger.debug(f"{self}: Reset internal state") + async def start(self, frame: StartFrame): """Initialize the service upon receiving a StartFrame.""" await super().start(frame) self._output_format = output_format_from_sample_rate(self.sample_rate) - self._cumulative_time = 0 - self._started = False - self._previous_text = "" + self._reset_state() async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): await super().push_frame(frame, direction) if isinstance(frame, (StartInterruptionFrame, TTSStoppedFrame)): # Reset timing on interruption or stop - self._started = False - self._cumulative_time = 0 - self._previous_text = "" + self._reset_state() if isinstance(frame, TTSStoppedFrame): await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)]) @@ -631,7 +634,6 @@ class ElevenLabsHttpTTSService(WordTTSService): # Include previous text as context if available if self._previous_text: payload["previous_text"] = self._previous_text - print(f"Previous text: {self._previous_text}") if self._voice_settings: payload["voice_settings"] = self._voice_settings