From 0df75b0915a31603f16e73bc779496d73e681829 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 8 Nov 2025 08:24:55 -0500 Subject: [PATCH 001/110] Add ar-XA language code for Gemini Live --- src/pipecat/transcriptions/language.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py index cd49c4645..cc84346c7 100644 --- a/src/pipecat/transcriptions/language.py +++ b/src/pipecat/transcriptions/language.py @@ -64,6 +64,7 @@ class Language(StrEnum): AR_SA = "ar-SA" AR_SY = "ar-SY" AR_TN = "ar-TN" + AR_XA = "ar-XA" AR_YE = "ar-YE" # Assamese From c38055dbdddd7f95e3a8a8717dd733e1fbe3e5c4 Mon Sep 17 00:00:00 2001 From: Julien Vantyghem Date: Sun, 9 Nov 2025 18:51:04 +0100 Subject: [PATCH 002/110] fix(deepgram-flux): urlencode keyterm and tag parameters --- src/pipecat/services/deepgram/flux/stt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index 9b22a5d28..7adc0f35a 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -9,6 +9,7 @@ import json from enum import Enum from typing import Any, AsyncGenerator, Dict, Optional +from urllib.parse import urlencode from loguru import logger from pydantic import BaseModel @@ -285,11 +286,11 @@ class DeepgramFluxSTTService(WebsocketSTTService): # Add keyterm parameters (can have multiple) for keyterm in self._params.keyterm: - url_params.append(f"keyterm={keyterm}") + url_params.append(urlencode({"keyterm": keyterm})) # Add tag parameters (can have multiple) for tag_value in self._params.tag: - url_params.append(f"tag={tag_value}") + url_params.append(urlencode({"tag": tag_value})) self._websocket_url = f"{self._url}?{'&'.join(url_params)}" await self._connect() From 2300941bb880d4af1bbe484848da01de7e45522c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 10 Nov 2025 09:55:46 -0500 Subject: [PATCH 003/110] Revert "Merge pull request #3004 from pipecat-ai/mb/improve-concat-aggregated-text" This reverts commit 5e7f59a0b03c840e05dc342b2988037578cb0f8f, reversing changes made to 2ad4122b77e2cb27145281c28991dc7af613b15b. --- CHANGELOG.md | 4 -- src/pipecat/utils/string.py | 40 ++---------- tests/test_transcript_processor.py | 100 ----------------------------- 3 files changed, 6 insertions(+), 138 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6394f744a..8443b2348 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,10 +90,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated `simli-ai` to 0.1.25. -- Improved `concatenate_aggregated_text()` to one word outputs from OpenAI - Realtime and Gemini Live. Text fragments are now correctly concatenated - without spaces when these patterns are detected. - - `STTMuteFilter` no longer sends `STTMuteFrame` to the STT service. The filter now blocks frames locally without instructing the STT service to stop processing audio. This prevents inactivity-related errors (such as 409 errors diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 298a09472..25ce6afd5 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -218,43 +218,15 @@ def concatenate_aggregated_text(text_parts: List[str]) -> str: has_leading_spaces = any(part and part[0] == " " for part in text_parts[1:]) has_trailing_spaces = any(part and part[-1] == " " for part in text_parts[:-1]) - # Check for trailing non-space whitespace (e.g., \n, \r, \t) which indicates - # syllable-by-syllable output with line breaks. - # Example: Gemini Live: ["Met", "amo", "rph", "osi", "s.\n"] - has_trailing_whitespace = any( - part and part[-1] != " " and part[-1].isspace() for part in text_parts - ) + # If there are embedded spaces in the fragments, use direct concatenation + contains_spacing_between_fragments = has_leading_spaces or has_trailing_spaces - # Check if we have punctuation-only fragments, which indicates syllable-by-syllable - # output where punctuation arrives as a separate fragment. - # Example: OpenAI Realtime single word: ["Met", "am", "orph", "osis", "."] - punctuation_chars = ".,!?;:—-'\"…" - has_punctuation_only = any( - part and len(part.strip()) == 0 or all(c in punctuation_chars for c in part) - for part in text_parts - ) - - # If there are embedded spaces or other whitespace in the fragments, use direct concatenation - contains_spacing_between_fragments = ( - has_leading_spaces or has_trailing_spaces or has_trailing_whitespace - ) - - # Apply corresponding joining method based on detected spacing patterns: - - if has_punctuation_only and not contains_spacing_between_fragments: - # Syllable-by-syllable output with standalone punctuation fragment. Examples: - # - OpenAI Realtime: ["Met", "am", "orph", "osis", "."] → "Metamorphosis." - result = "".join(text_parts) - elif contains_spacing_between_fragments: - # Fragments already have embedded spacing or trailing whitespace - concatenate directly. Examples: - # - OpenAI Realtime: ['Hey', ' there', '!', ' Great', ' to', ' meet', ' you', '!'] - # - Gemini Live (spaces): ['Hel', 'lo.', ' Wo', 'u', 'ld ', 'you', ' li', 'ke ', 'to ', 'he', 'ar a joke?\n'] - # - Gemini Live (newline): ["Met", "amo", "rph", "osi", "s.\n"] → "Metamorphosis." - # - Sentence level TTS services: ['Hello!', ' How can I assist you today?'] + # Apply corresponding joining method + if contains_spacing_between_fragments: + # Fragments already have spacing - just concatenate result = "".join(text_parts) else: - # Word-by-word fragments without spacing - join with spaces. Examples: - # - Word level TTS services: ["Hello", "there.", "How", "are", "you?"] → "Hello there. How are you?" + # Word-by-word fragments - join with spaces result = " ".join(text_parts) # Clean up any excessive whitespace diff --git a/tests/test_transcript_processor.py b/tests/test_transcript_processor.py index d45d5ba3b..b433951ce 100644 --- a/tests/test_transcript_processor.py +++ b/tests/test_transcript_processor.py @@ -479,103 +479,3 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): self.assertEqual(message.role, "assistant") # Should be properly joined without extra spaces self.assertEqual(message.content, "Hello there! How's it going?") - - async def test_openai_realtime_syllable_fragments(self): - """Test OpenAI Realtime syllable-by-syllable output with standalone punctuation - - OpenAI Realtime can output single words as syllable fragments with punctuation - as a separate fragment. Example: ["Met", "am", "orph", "osis", "."] - This should be concatenated without spaces to form "Metamorphosis." - """ - processor = AssistantTranscriptProcessor() - - received_updates = [] - - @processor.event_handler("on_transcript_update") - async def handle_update(proc, frame: TranscriptionUpdateFrame): - received_updates.append(frame) - - # Simulate OpenAI Realtime syllable-by-syllable output - frames_to_send = [ - BotStartedSpeakingFrame(), - SleepFrame(), - TTSTextFrame(text="Met"), - TTSTextFrame(text="am"), - TTSTextFrame(text="orph"), - TTSTextFrame(text="osis"), - TTSTextFrame(text="."), # Standalone punctuation fragment - BotStoppedSpeakingFrame(), - ] - - expected_down_frames = [ - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TranscriptionUpdateFrame, - ] - - await run_test( - processor, - frames_to_send=frames_to_send, - expected_down_frames=expected_down_frames, - ) - - # Verify syllables are concatenated without spaces - self.assertEqual(len(received_updates), 1) - message = received_updates[0].messages[0] - self.assertEqual(message.role, "assistant") - self.assertEqual(message.content, "Metamorphosis.") - - async def test_gemini_live_syllable_fragments_with_newline(self): - """Test Gemini Live syllable-by-syllable output with trailing newline - - Gemini Live can output syllable fragments where the last fragment contains - trailing whitespace like newlines. Example: ["Met", "amo", "rph", "osi", "s.\\n"] - This should be concatenated without spaces to form "Metamorphosis." - """ - processor = AssistantTranscriptProcessor() - - received_updates = [] - - @processor.event_handler("on_transcript_update") - async def handle_update(proc, frame: TranscriptionUpdateFrame): - received_updates.append(frame) - - # Simulate Gemini Live syllable-by-syllable output with trailing newline - frames_to_send = [ - BotStartedSpeakingFrame(), - SleepFrame(), - TTSTextFrame(text="Met"), - TTSTextFrame(text="amo"), - TTSTextFrame(text="rph"), - TTSTextFrame(text="osi"), - TTSTextFrame(text="s.\n"), # Last fragment with trailing newline - BotStoppedSpeakingFrame(), - ] - - expected_down_frames = [ - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TranscriptionUpdateFrame, - ] - - await run_test( - processor, - frames_to_send=frames_to_send, - expected_down_frames=expected_down_frames, - ) - - # Verify syllables are concatenated without spaces and newline is stripped - self.assertEqual(len(received_updates), 1) - message = received_updates[0].messages[0] - self.assertEqual(message.role, "assistant") - self.assertEqual(message.content, "Metamorphosis.") From c1c7a561ede756f6c7311f4042b1640f916771de Mon Sep 17 00:00:00 2001 From: vipyne Date: Mon, 10 Nov 2025 11:06:12 -0600 Subject: [PATCH 004/110] remove LivekitFrameSerializer --- CHANGELOG.md | 6 ++ src/pipecat/serializers/livekit.py | 98 ------------------------------ 2 files changed, 6 insertions(+), 98 deletions(-) delete mode 100644 src/pipecat/serializers/livekit.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8443b2348..7a2d000b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Not released yet + +### Removed + +- `LivekitFrameSerializer` has been removed. Use `LiveKitTransport` instead. + ## [0.0.93] - 2025-11-07 ### Added diff --git a/src/pipecat/serializers/livekit.py b/src/pipecat/serializers/livekit.py deleted file mode 100644 index f3a34c434..000000000 --- a/src/pipecat/serializers/livekit.py +++ /dev/null @@ -1,98 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -"""LiveKit frame serializer for Pipecat.""" - -import ctypes -import pickle - -from loguru import logger - -from pipecat.frames.frames import Frame, InputAudioRawFrame, OutputAudioRawFrame -from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType - -try: - from livekit.rtc import AudioFrame -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error("In order to use LiveKit, you need to `pip install pipecat-ai[livekit]`.") - raise Exception(f"Missing module: {e}") - - -class LivekitFrameSerializer(FrameSerializer): - """Serializer for converting between Pipecat frames and LiveKit audio frames. - - .. deprecated:: 0.0.90 - - This class is deprecated and will be removed in a future version. - Please use LiveKitTransport instead, which handles audio streaming - and frame conversion natively. - - This serializer handles the conversion of Pipecat's OutputAudioRawFrame objects - to LiveKit AudioFrame objects for transmission, and the reverse conversion - for received audio data. - """ - - def __init__(self): - """Initialize the LiveKit frame serializer.""" - super().__init__() - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "LivekitFrameSerializer is deprecated and will be removed in a future version. " - "Please use LiveKitTransport instead, which handles audio streaming natively.", - DeprecationWarning, - stacklevel=2, - ) - - @property - def type(self) -> FrameSerializerType: - """Get the serializer type. - - Returns: - The serializer type indicating binary serialization. - """ - return FrameSerializerType.BINARY - - async def serialize(self, frame: Frame) -> str | bytes | None: - """Serialize a Pipecat frame to LiveKit AudioFrame format. - - Args: - frame: The Pipecat frame to serialize. Only OutputAudioRawFrame - instances are supported. - - Returns: - Pickled LiveKit AudioFrame bytes if frame is OutputAudioRawFrame, - None otherwise. - """ - if not isinstance(frame, OutputAudioRawFrame): - return None - audio_frame = AudioFrame( - data=frame.audio, - sample_rate=frame.sample_rate, - num_channels=frame.num_channels, - samples_per_channel=len(frame.audio) // ctypes.sizeof(ctypes.c_int16), - ) - return pickle.dumps(audio_frame) - - async def deserialize(self, data: str | bytes) -> Frame | None: - """Deserialize LiveKit AudioFrame data to a Pipecat frame. - - Args: - data: Pickled data containing a LiveKit AudioFrame. - - Returns: - InputAudioRawFrame containing the deserialized audio data, - or None if deserialization fails. - """ - audio_frame: AudioFrame = pickle.loads(data)["frame"] - return InputAudioRawFrame( - audio=bytes(audio_frame.data), - sample_rate=audio_frame.sample_rate, - num_channels=audio_frame.num_channels, - ) From 913194844e430a3056bd1272e87f450d42706499 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 10 Nov 2025 12:28:40 -0500 Subject: [PATCH 005/110] Make the mechanism of adding spaces when concatenating TTS (or speech-to-speech LLM) output text explicit and deterministic, rather than heuristic-based. This fixes a bug where spaces were sometimes missing from assistant messages in context. --- CHANGELOG.md | 5 ++ src/pipecat/frames/frames.py | 8 +++ .../aggregators/llm_response_universal.py | 14 ++++- .../processors/frameworks/langchain.py | 4 +- .../processors/transcript_processor.py | 12 ++++- .../services/google/gemini_live/llm.py | 11 +++- src/pipecat/services/openai/realtime/llm.py | 10 +++- src/pipecat/utils/string.py | 21 ++------ tests/test_context_aggregators.py | 51 ++++++++++++++----- tests/test_transcript_processor.py | 21 +++++--- 10 files changed, 113 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a2d000b0..afa2a5651 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `LivekitFrameSerializer` has been removed. Use `LiveKitTransport` instead. +### Fixed + +- Fixed a bug related to `LLMAssistantAggregator` where spaces were sometimes + missing from assistant messages in context. + ## [0.0.93] - 2025-11-07 ### Added diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 3e92b8480..6f48f79f7 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -330,10 +330,18 @@ class TextFrame(DataFrame): text: str skip_tts: bool = field(init=False) + # Whether any necessary inter-frame (leading/trailing) spaces are already + # included in the text. + # NOTE: Ideally this would be available at init time with a default value, + # but that would impact how subclasses can be initialized (it would require + # mandatory fields of theirs to have defaults to preserve + # non-default-before-default argument order) + includes_inter_frame_spaces: bool = field(init=False) def __post_init__(self): super().__post_init__() self.skip_tts = False + self.includes_inter_frame_spaces = False def __str__(self): pts = format_pts(self.pts) diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 882428a6e..d4d9ad7da 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -92,6 +92,14 @@ class LLMContextAggregator(FrameProcessor): self._aggregation: List[str] = [] + # Whether to add spaces between text parts. + # (Currently only used by LLMAssistantAggregator, but could be expanded + # to LLMUserAggregator in the future if needed; that would require + # additional work since LLMUserAggregator currently trims spaces from + # incoming frames before determining whether it "really" received any + # text). + self._add_spaces = True + @property def messages(self) -> List[LLMContextMessage]: """Get messages from the LLM context. @@ -183,7 +191,7 @@ class LLMContextAggregator(FrameProcessor): Returns: The concatenated aggregation string. """ - return concatenate_aggregated_text(self._aggregation) + return concatenate_aggregated_text(self._aggregation, self._add_spaces) class LLMUserAggregator(LLMContextAggregator): @@ -813,6 +821,10 @@ class LLMAssistantAggregator(LLMContextAggregator): if len(frame.text) == 0: return + # Track whether we need to add spaces between text parts + # Assumption: we can just keep track of the latest frame's value + self._add_spaces = not frame.includes_inter_frame_spaces + self._aggregation.append(frame.text) def _context_updated_task_finished(self, task: asyncio.Task): diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index 97a6ce343..b8a472a3d 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -107,7 +107,9 @@ class LangchainProcessor(FrameProcessor): {self._transcript_key: text}, config={"configurable": {"session_id": self._participant_id}}, ): - await self.push_frame(TextFrame(self.__get_token_value(token))) + frame = TextFrame(self.__get_token_value(token)) + frame.includes_inter_frame_spaces = True + await self.push_frame(frame) except GeneratorExit: logger.warning(f"{self} generator was closed prematurely") except Exception as e: diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 13b2bb97f..0b6f1b5bb 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -101,6 +101,12 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): self._current_text_parts: List[str] = [] self._aggregation_start_time: Optional[str] = None + # Whether to add spaces between text parts. + # (The use of this could be expanded to the UserTranscriptProcessor in + # the future if needed; currently the UserTranscriptProcessor assumes + # that user transcription frames do not need aggregation). + self._add_spaces = True + async def _emit_aggregated_text(self): """Aggregates and emits text fragments as a transcript message. @@ -141,7 +147,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): Result: "Hello there how are you" """ if self._current_text_parts and self._aggregation_start_time: - content = concatenate_aggregated_text(self._current_text_parts) + content = concatenate_aggregated_text(self._current_text_parts, self._add_spaces) if content: logger.trace(f"Emitting aggregated assistant message: {content}") message = TranscriptionMessage( @@ -185,6 +191,10 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): if not self._aggregation_start_time: self._aggregation_start_time = time_now_iso8601() + # Track whether we need to add spaces between text parts + # Assumption: we can just keep track of the latest frame's value + self._add_spaces = not frame.includes_inter_frame_spaces + self._current_text_parts.append(frame.text) # Push frame. diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 0f3304529..9c92076ab 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -1453,7 +1453,10 @@ class GeminiLiveLLMService(LLMService): self._bot_text_buffer += text self._search_result_buffer += text # Also accumulate for grounding - await self.push_frame(LLMTextFrame(text=text)) + frame = LLMTextFrame(text=text) + # Gemini Live text already includes any necessary inter-chunk spaces + frame.includes_inter_frame_spaces = True + await self.push_frame(frame) # Check for grounding metadata in server content if msg.server_content and msg.server_content.grounding_metadata: @@ -1645,7 +1648,11 @@ class GeminiLiveLLMService(LLMService): await self.push_frame(TTSStartedFrame()) await self.push_frame(LLMFullResponseStartFrame()) - await self.push_frame(TTSTextFrame(text=text)) + frame = TTSTextFrame(text=text) + # Gemini Live text already includes any necessary inter-chunk spaces + frame.includes_inter_frame_spaces = True + + await self.push_frame(frame) async def _handle_msg_grounding_metadata(self, message: LiveServerMessage): """Handle dedicated grounding metadata messages.""" diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 0129a94f9..e38836b90 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -679,13 +679,19 @@ class OpenAIRealtimeLLMService(LLMService): # We receive text deltas (as opposed to audio transcript deltas) when # the output modality is "text" if evt.delta: - await self.push_frame(LLMTextFrame(evt.delta)) + frame = LLMTextFrame(evt.delta) + # OpenAI Realtime text already includes any necessary inter-chunk spaces + frame.includes_inter_frame_spaces = True + await self.push_frame(frame) async def _handle_evt_audio_transcript_delta(self, evt): # We receive audio transcript deltas (as opposed to text deltas) when # the output modality is "audio" (the default) if evt.delta: - await self.push_frame(TTSTextFrame(evt.delta)) + frame = TTSTextFrame(evt.delta) + # OpenAI Realtime text already includes any necessary inter-chunk spaces + frame.includes_inter_frame_spaces = True + await self.push_frame(frame) async def _handle_evt_function_call_arguments_done(self, evt): """Handle completion of function call arguments. diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 25ce6afd5..177c0e8a5 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -198,7 +198,7 @@ def parse_start_end_tags( return (None, current_tag_index) -def concatenate_aggregated_text(text_parts: List[str]) -> str: +def concatenate_aggregated_text(text_parts: List[str], add_spaces: bool) -> str: """Concatenate a list of text parts into a single string. This function joins the provided list of text parts into a single string, @@ -209,25 +209,14 @@ def concatenate_aggregated_text(text_parts: List[str]) -> str: Args: text_parts: A list of strings representing parts of text to concatenate. + add_spaces: Whether to add spaces between text parts during concatenation. Returns: A single concatenated string. """ - # Check specifically for space characters, previously isspace() was used - # but that includes all whitespace characters (e.g. \n), not just spaces. - has_leading_spaces = any(part and part[0] == " " for part in text_parts[1:]) - has_trailing_spaces = any(part and part[-1] == " " for part in text_parts[:-1]) - - # If there are embedded spaces in the fragments, use direct concatenation - contains_spacing_between_fragments = has_leading_spaces or has_trailing_spaces - - # Apply corresponding joining method - if contains_spacing_between_fragments: - # Fragments already have spacing - just concatenate - result = "".join(text_parts) - else: - # Word-by-word fragments - join with spaces - result = " ".join(text_parts) + # Concatenate text parts with or without spaces based on the flag + separator = " " if add_spaces else "" + result = separator.join(text_parts) # Clean up any excessive whitespace result = result.strip() diff --git a/tests/test_context_aggregators.py b/tests/test_context_aggregators.py index 6196032a3..12be482c1 100644 --- a/tests/test_context_aggregators.py +++ b/tests/test_context_aggregators.py @@ -35,6 +35,7 @@ from pipecat.frames.frames import ( ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.task import PipelineParams +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import ( LLMAssistantAggregatorParams, LLMUserAggregatorParams, @@ -651,12 +652,20 @@ class BaseTestAssistantContextAggregator: aggregator = self.AGGREGATOR_CLASS( context, params=self.create_assistant_aggregator_params(expect_stripped_words=False) ) + + # The newer LLMAssistantAggregator expects TextFrames to declare + # when they include inter-frame spaces. + def make_text_frame(text: str) -> TextFrame: + frame = TextFrame(text=text) + frame.includes_inter_frame_spaces = True + return frame + frames_to_send = [ LLMFullResponseStartFrame(), - TextFrame(text="Hello "), - TextFrame(text="Pipecat. "), - TextFrame(text="How are "), - TextFrame(text="you?"), + make_text_frame("Hello "), + make_text_frame("Pipecat. "), + make_text_frame("How are "), + make_text_frame("you?"), LLMFullResponseEndFrame(), ] expected_down_frames = [*self.EXPECTED_CONTEXT_FRAMES] @@ -697,14 +706,22 @@ class BaseTestAssistantContextAggregator: aggregator = self.AGGREGATOR_CLASS( context, params=self.create_assistant_aggregator_params(expect_stripped_words=False) ) + + # The newer LLMAssistantAggregator expects TextFrames to declare + # when they include inter-frame spaces. + def make_text_frame(text: str) -> TextFrame: + frame = TextFrame(text=text) + frame.includes_inter_frame_spaces = True + return frame + frames_to_send = [ LLMFullResponseStartFrame(), - TextFrame(text="Hello "), - TextFrame(text="Pipecat."), + make_text_frame("Hello "), + make_text_frame("Pipecat."), LLMFullResponseEndFrame(), LLMFullResponseStartFrame(), - TextFrame(text="How are "), - TextFrame(text="you?"), + make_text_frame(text="How are "), + make_text_frame(text="you?"), LLMFullResponseEndFrame(), ] expected_down_frames = [*self.EXPECTED_CONTEXT_FRAMES, *self.EXPECTED_CONTEXT_FRAMES] @@ -724,16 +741,24 @@ class BaseTestAssistantContextAggregator: aggregator = self.AGGREGATOR_CLASS( context, params=self.create_assistant_aggregator_params(expect_stripped_words=False) ) + + # The newer LLMAssistantAggregator expects TextFrames to declare + # when they include inter-frame spaces. + def make_text_frame(text: str) -> TextFrame: + frame = TextFrame(text=text) + frame.includes_inter_frame_spaces = True + return frame + frames_to_send = [ LLMFullResponseStartFrame(), - TextFrame(text="Hello "), - TextFrame(text="Pipecat."), + make_text_frame("Hello "), + make_text_frame("Pipecat."), LLMFullResponseEndFrame(), SleepFrame(AGGREGATION_SLEEP), InterruptionFrame(), LLMFullResponseStartFrame(), - TextFrame(text="How are "), - TextFrame(text="you?"), + make_text_frame("How are "), + make_text_frame("you?"), LLMFullResponseEndFrame(), ] expected_down_frames = [ @@ -969,7 +994,7 @@ class TestOpenAIAssistantContextAggregator( class TestLLMAssistantAggregator( BaseTestAssistantContextAggregator, unittest.IsolatedAsyncioTestCase ): - CONTEXT_CLASS = OpenAILLMContext + CONTEXT_CLASS = LLMContext AGGREGATOR_CLASS = LLMAssistantAggregator EXPECTED_CONTEXT_FRAMES = [LLMContextFrame, LLMContextAssistantTimestampFrame] diff --git a/tests/test_transcript_processor.py b/tests/test_transcript_processor.py index b433951ce..19366086c 100644 --- a/tests/test_transcript_processor.py +++ b/tests/test_transcript_processor.py @@ -438,17 +438,22 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): received_updates.append(frame) # Test the specific pattern shared + def make_tts_text_frame(text: str) -> TTSTextFrame: + frame = TTSTextFrame(text=text) + frame.includes_inter_frame_spaces = True + return frame + frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), - TTSTextFrame(text="Hello"), - TTSTextFrame(text=" there"), - TTSTextFrame(text="!"), - TTSTextFrame(text=" How"), - TTSTextFrame(text="'s"), - TTSTextFrame(text=" it"), - TTSTextFrame(text=" going"), - TTSTextFrame(text="?"), + make_tts_text_frame("Hello"), + make_tts_text_frame(" there"), + make_tts_text_frame("!"), + make_tts_text_frame(" How"), + make_tts_text_frame("'s"), + make_tts_text_frame(" it"), + make_tts_text_frame(" going"), + make_tts_text_frame("?"), BotStoppedSpeakingFrame(), ] From 588dcf2ab94eb14c49dadd4457d28b8acda4de20 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 10 Nov 2025 14:29:54 -0500 Subject: [PATCH 006/110] Add Sarvam STT to README list --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b100bf220..ffe826025 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout | Category | Services | | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | | Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | | Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | From 8dd45af5b7ba7eeeed0975629761282f8d02806d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 10 Nov 2025 09:43:49 -0500 Subject: [PATCH 007/110] Deprecate KrispFilter --- CHANGELOG.md | 7 ++++++- src/pipecat/audio/filters/krisp_filter.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afa2a5651..ced4af28b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,12 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Not released yet +## [Unreleased] + +### Deprecated + +- The KrispFilter is deprecated and will be removed in a future version. Use + the KrispVivaFilter instead. ### Removed diff --git a/src/pipecat/audio/filters/krisp_filter.py b/src/pipecat/audio/filters/krisp_filter.py index 267d2f2ea..61568ace6 100644 --- a/src/pipecat/audio/filters/krisp_filter.py +++ b/src/pipecat/audio/filters/krisp_filter.py @@ -61,6 +61,10 @@ class KrispFilter(BaseAudioFilter): Provides real-time noise reduction for audio streams using Krisp's proprietary noise suppression algorithms. Requires a Krisp model file for operation. + + .. deprecated:: 0.0.94 + The KrispFilter is deprecated and will be removed in a future version. + Use KrispVivaFilter instead. """ def __init__( @@ -79,6 +83,17 @@ class KrispFilter(BaseAudioFilter): """ super().__init__() + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "KrispFilter is deprecated and will be removed in a future version. " + "Use KrispVivaFilter instead.", + DeprecationWarning, + stacklevel=2, + ) + # Set model path, checking environment if not specified self._model_path = model_path or os.getenv("KRISP_MODEL_PATH") if not self._model_path: From ee494918a980b6cc7b716617a991c83c3b022795 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 10 Nov 2025 16:18:47 -0500 Subject: [PATCH 008/110] Prep for 0.0.94 hotfix --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ced4af28b..fc37064e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,12 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.94] - 2025-11-10 ### Deprecated -- The KrispFilter is deprecated and will be removed in a future version. Use - the KrispVivaFilter instead. +- The `KrispFilter` is deprecated and will be removed in a future version. Use + the `KrispVivaFilter` instead. ### Removed From adf5198423fbb825751ad01178cdbde259fafade Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Tue, 11 Nov 2025 13:49:14 +0000 Subject: [PATCH 009/110] Support for retry when 503 error to TTS API. --- CHANGELOG.md | 5 ++ src/pipecat/services/speechmatics/tts.py | 99 +++++++++++++++--------- 2 files changed, 68 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc37064e4..fb7002fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.0.94] - 2025-11-10 +### Changed + +- Added support for retrying `SpeechmaticsTTSService` when it returns a 503 + error. Default values in `InputParams`. + ### Deprecated - The `KrispFilter` is deprecated and will be removed in a future version. Use diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index 23d10c5e1..a99433f73 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -6,6 +6,7 @@ """Speechmatics TTS service integration.""" +import asyncio from typing import AsyncGenerator, Optional from urllib.parse import urlencode @@ -45,7 +46,8 @@ class SpeechmaticsTTSService(TTSService): class InputParams(BaseModel): """Optional input parameters for Speechmatics TTS configuration.""" - pass + retry_interval_s: float = 0.02 + retry_timeout_s: float = 1.0 def __init__( self, @@ -125,46 +127,71 @@ class SpeechmaticsTTSService(TTSService): try: await self.start_ttfb_metrics() - async with self._session.post(url, json=payload, headers=headers) as response: - if response.status != 200: - error_message = f"Speechmatics TTS error: HTTP {response.status}" - logger.error(error_message) - yield ErrorFrame(error=error_message) - return + # Retry loop for 503 responses + start_time = asyncio.get_event_loop().time() - await self.start_tts_usage_metrics(text) + while True: + async with self._session.post(url, json=payload, headers=headers) as response: + if response.status == 503: + elapsed_time = asyncio.get_event_loop().time() - start_time + if elapsed_time >= self._params.retry_timeout_s: + error_message = ( + f"{self} HTTP 503 (timeout after {self._params.retry_timeout_s}s)" + ) + logger.error(error_message) + yield ErrorFrame(error=error_message) + return - yield TTSStartedFrame() - - # Process the response in streaming chunks - first_chunk = True - buffer = b"" - - async for chunk in response.content.iter_any(): - if not chunk: - continue - if first_chunk: - await self.stop_ttfb_metrics() - first_chunk = False - - buffer += chunk - - # Emit all complete 2-byte int16 samples from buffer - if len(buffer) >= 2: - complete_samples = len(buffer) // 2 - complete_bytes = complete_samples * 2 - - audio_data = buffer[:complete_bytes] - buffer = buffer[complete_bytes:] # Keep remaining bytes for next iteration - - yield TTSAudioRawFrame( - audio=audio_data, - sample_rate=self.sample_rate, - num_channels=1, + logger.debug( + f"{self} Received 503, retrying in {self._params.retry_interval_s}s..." ) + await asyncio.sleep(self._params.retry_interval_s) + continue + + if response.status != 200: + error_message = f"{self} HTTP {response.status}" + logger.error(error_message) + yield ErrorFrame(error=error_message) + return + + await self.start_tts_usage_metrics(text) + + yield TTSStartedFrame() + + # Process the response in streaming chunks + first_chunk = True + buffer = b"" + + async for chunk in response.content.iter_any(): + if not chunk: + continue + if first_chunk: + await self.stop_ttfb_metrics() + first_chunk = False + + buffer += chunk + + # Emit all complete 2-byte int16 samples from buffer + if len(buffer) >= 2: + complete_samples = len(buffer) // 2 + complete_bytes = complete_samples * 2 + + audio_data = buffer[:complete_bytes] + buffer = buffer[ + complete_bytes: + ] # Keep remaining bytes for next iteration + + yield TTSAudioRawFrame( + audio=audio_data, + sample_rate=self.sample_rate, + num_channels=1, + ) + + # Successfully processed the response, break out of retry loop + break except Exception as e: - logger.exception(f"Error generating TTS: {e}") + logger.exception(f"{self}: Error generating TTS: {e}") yield ErrorFrame(error=f"Speechmatics TTS error: {str(e)}") finally: yield TTSStoppedFrame() From 41ac43cf71f91b2ca751dcc9a4cef57af8ff0f96 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Tue, 11 Nov 2025 13:56:45 +0000 Subject: [PATCH 010/110] updated docs --- src/pipecat/services/speechmatics/tts.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index a99433f73..3086aaa44 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -44,7 +44,12 @@ class SpeechmaticsTTSService(TTSService): SPEECHMATICS_SAMPLE_RATE = 16000 class InputParams(BaseModel): - """Optional input parameters for Speechmatics TTS configuration.""" + """Optional input parameters for Speechmatics TTS configuration. + + Parameters: + retry_interval_s: Interval between retries in seconds. Defaults to 0.02. + retry_timeout_s: Timeout for retries in seconds. Defaults to 1.0. + """ retry_interval_s: float = 0.02 retry_timeout_s: float = 1.0 From 62caadfc7c45bd428fea8863b938bcd1a1b006ab Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 11 Nov 2025 11:37:46 -0300 Subject: [PATCH 011/110] Preventing HeyGenVideoService from disconnecting. --- src/pipecat/services/heygen/client.py | 14 ++++++++++++-- src/pipecat/services/heygen/video.py | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/heygen/client.py b/src/pipecat/services/heygen/client.py index 1b464ce9c..e142aab04 100644 --- a/src/pipecat/services/heygen/client.py +++ b/src/pipecat/services/heygen/client.py @@ -83,6 +83,7 @@ class HeyGenClient: version="v2", ), callbacks: HeyGenCallbacks, + connect_as_user: bool = False, ) -> None: """Initialize the HeyGen client. @@ -92,6 +93,7 @@ class HeyGenClient: params: Transport configuration parameters session_request: Configuration for the HeyGen session (default: uses Shawn_Therapist_public avatar) callbacks: Callback handlers for HeyGen events + connect_as_user: Whether to connect using the user token or not (default: False) """ self._api = HeyGenApi(api_key, session=session) self._heyGen_session: Optional[HeyGenSession] = None @@ -119,6 +121,11 @@ class HeyGenClient: self._next_send_time = 0 self._audio_seconds_sent = 0.0 self._transport_ready = False + # HeyGen enforces a protection mechanism that will automatically disconnect the avatar if a user does not join within 5 minutes, + # regardless of whether the Pipecat agent remains present in the room. + # To prevent unexpected disconnections in HeyGenVideoService, we ensure that a user connection is established using the user's token. + # This keeps the avatar session active and avoids forced logouts due to inactivity from the user side. + self._connect_as_user = connect_as_user async def _initialize(self): self._heyGen_session = await self._api.new_session(self._session_request) @@ -562,9 +569,12 @@ class HeyGenClient: self._callbacks.on_participant_disconnected, participant.identity ) - await self._livekit_room.connect( - self._heyGen_session.url, self._heyGen_session.livekit_agent_token + access_token = ( + self._heyGen_session.livekit_agent_token + if not self._connect_as_user + else self._heyGen_session.access_token ) + await self._livekit_room.connect(self._heyGen_session.url, access_token) logger.debug(f"Successfully connected to LiveKit room: {self._livekit_room.name}") logger.debug(f"Local participant SID: {self._livekit_room.local_participant.sid}") logger.debug( diff --git a/src/pipecat/services/heygen/video.py b/src/pipecat/services/heygen/video.py index bf7fbeecd..b2df15119 100644 --- a/src/pipecat/services/heygen/video.py +++ b/src/pipecat/services/heygen/video.py @@ -121,6 +121,7 @@ class HeyGenVideoService(AIService): on_participant_connected=self._on_participant_connected, on_participant_disconnected=self._on_participant_disconnected, ), + connect_as_user=True, ) await self._client.setup(setup) From b76b25a6e1a6d1ce3a154189a59c885bcfcb5295 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 11 Nov 2025 11:58:31 -0300 Subject: [PATCH 012/110] Mentioning the HeyGen fix in the changelog. --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc37064e4..974133011 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Fixed + +- Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. + ## [0.0.94] - 2025-11-10 ### Deprecated From 0febfc62ec1854327c5ccd4f7adee2891513ce66 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Tue, 11 Nov 2025 17:45:22 +0000 Subject: [PATCH 013/110] Updated to use backoff utility function. --- src/pipecat/services/speechmatics/tts.py | 85 +++++++++++++++--------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index 3086aaa44..7b40cf4eb 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -16,12 +16,14 @@ from pydantic import BaseModel from pipecat.frames.frames import ( ErrorFrame, + FatalErrorFrame, Frame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, ) from pipecat.services.tts_service import TTSService +from pipecat.utils.network import exponential_backoff_time from pipecat.utils.tracing.service_decorators import traced_tts try: @@ -44,15 +46,9 @@ class SpeechmaticsTTSService(TTSService): SPEECHMATICS_SAMPLE_RATE = 16000 class InputParams(BaseModel): - """Optional input parameters for Speechmatics TTS configuration. + """Optional input parameters for Speechmatics TTS configuration.""" - Parameters: - retry_interval_s: Interval between retries in seconds. Defaults to 0.02. - retry_timeout_s: Timeout for retries in seconds. Defaults to 1.0. - """ - - retry_interval_s: float = 0.02 - retry_timeout_s: float = 1.0 + pass def __init__( self, @@ -116,57 +112,87 @@ class SpeechmaticsTTSService(TTSService): Yields: Frame: Audio frames containing the synthesized speech. """ + # Log the TTS started frame logger.debug(f"{self}: Generating TTS [{text}]") + # HTTP headers headers = { "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json", } + # HTTP payload payload = { "text": text, } + # Complete HTTP URL url = _get_endpoint_url(self._base_url, self._voice_id, self.sample_rate) try: + # Start TTS TTFB metrics await self.start_ttfb_metrics() - # Retry loop for 503 responses - start_time = asyncio.get_event_loop().time() + # Track attempt + attempt = 0 + # Keep retrying until we get a 200 response or timeout while True: async with self._session.post(url, json=payload, headers=headers) as response: + """Evaluate response from TTS service.""" + + # 503 : Service unavailable if response.status == 503: - elapsed_time = asyncio.get_event_loop().time() - start_time - if elapsed_time >= self._params.retry_timeout_s: - error_message = ( - f"{self} HTTP 503 (timeout after {self._params.retry_timeout_s}s)" + """Calculate the backoff time and retry.""" + + try: + # Calculate the backoff time + backoff_time = exponential_backoff_time( + attempt=attempt, min_wait=0.25, max_wait=8.0, multiplier=0.5 + ) + + # Check if we've exceeded the maximum number of attempts + if backoff_time >= 8.0: + raise ValueError() + + # Report error frame + yield ErrorFrame( + error=f"{self} HTTP 503 (attempt {attempt}, retry in {backoff_time:.2f}s)" + ) + + # Wait before retrying + await asyncio.sleep(backoff_time) + + # Increment attempt + attempt += 1 + + # Retry + continue + + except (ValueError, ArithmeticError): + yield FatalErrorFrame( + error=f"{self} Service unavailable (attempts {attempt})" ) - logger.error(error_message) - yield ErrorFrame(error=error_message) return - logger.debug( - f"{self} Received 503, retrying in {self._params.retry_interval_s}s..." - ) - await asyncio.sleep(self._params.retry_interval_s) - continue - + # != 200 : Error if response.status != 200: - error_message = f"{self} HTTP {response.status}" - logger.error(error_message) - yield ErrorFrame(error=error_message) + yield FatalErrorFrame( + error=f"{self} Service unavailable ({response.status})" + ) return + # Update Pipecat metrics await self.start_tts_usage_metrics(text) + # Emit the TTS started frame yield TTSStartedFrame() # Process the response in streaming chunks first_chunk = True buffer = b"" + # Iterate over each audio data chunk from the TTS API async for chunk in response.content.iter_any(): if not chunk: continue @@ -182,10 +208,9 @@ class SpeechmaticsTTSService(TTSService): complete_bytes = complete_samples * 2 audio_data = buffer[:complete_bytes] - buffer = buffer[ - complete_bytes: - ] # Keep remaining bytes for next iteration + buffer = buffer[complete_bytes:] + # Emit the audio frame yield TTSAudioRawFrame( audio=audio_data, sample_rate=self.sample_rate, @@ -196,9 +221,9 @@ class SpeechmaticsTTSService(TTSService): break except Exception as e: - logger.exception(f"{self}: Error generating TTS: {e}") - yield ErrorFrame(error=f"Speechmatics TTS error: {str(e)}") + yield ErrorFrame(error=f"{self}: Error generating TTS: {e}") finally: + # Emit the TTS stopped frame yield TTSStoppedFrame() From 60bc77c7959031c865db80dbab65686706f0441b Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Tue, 11 Nov 2025 17:50:06 +0000 Subject: [PATCH 014/110] Update debugging messages. --- src/pipecat/services/speechmatics/tts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index 7b40cf4eb..455eb1ce0 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -157,7 +157,7 @@ class SpeechmaticsTTSService(TTSService): # Report error frame yield ErrorFrame( - error=f"{self} HTTP 503 (attempt {attempt}, retry in {backoff_time:.2f}s)" + error=f"{self} Service unavailable [503] (attempt {attempt}, retry in {backoff_time:.2f}s)" ) # Wait before retrying @@ -171,14 +171,14 @@ class SpeechmaticsTTSService(TTSService): except (ValueError, ArithmeticError): yield FatalErrorFrame( - error=f"{self} Service unavailable (attempts {attempt})" + error=f"{self} Service unavailable [503] (attempts {attempt})" ) return # != 200 : Error if response.status != 200: yield FatalErrorFrame( - error=f"{self} Service unavailable ({response.status})" + error=f"{self} Service unavailable [{response.status}]" ) return From 501744d7da8b66d06ac0e1a77612a1a06e3fd106 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Tue, 11 Nov 2025 17:53:31 +0000 Subject: [PATCH 015/110] Update CHANGELOG. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb7002fd6..df2c278de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Added support for retrying `SpeechmaticsTTSService` when it returns a 503 - error. Default values in `InputParams`. + error. ### Deprecated From 41cf9adef45fe4bf63d7c1be46bb333921bde0a4 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Tue, 11 Nov 2025 18:00:27 +0000 Subject: [PATCH 016/110] Updated for max retries. --- CHANGELOG.md | 2 +- src/pipecat/services/speechmatics/tts.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df2c278de..fb7002fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Added support for retrying `SpeechmaticsTTSService` when it returns a 503 - error. + error. Default values in `InputParams`. ### Deprecated diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index 455eb1ce0..b45c63ef7 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -46,9 +46,13 @@ class SpeechmaticsTTSService(TTSService): SPEECHMATICS_SAMPLE_RATE = 16000 class InputParams(BaseModel): - """Optional input parameters for Speechmatics TTS configuration.""" + """Optional input parameters for Speechmatics TTS configuration. - pass + Parameters: + max_retries: Maximum number of retries for TTS requests. Defaults to 5. + """ + + max_retries: int = 5 def __init__( self, @@ -151,8 +155,11 @@ class SpeechmaticsTTSService(TTSService): attempt=attempt, min_wait=0.25, max_wait=8.0, multiplier=0.5 ) + # Increment attempt + attempt += 1 + # Check if we've exceeded the maximum number of attempts - if backoff_time >= 8.0: + if attempt > self._params.max_retries: raise ValueError() # Report error frame @@ -163,9 +170,6 @@ class SpeechmaticsTTSService(TTSService): # Wait before retrying await asyncio.sleep(backoff_time) - # Increment attempt - attempt += 1 - # Retry continue From 217d7e9953de9af2cb2fe692a3522fbfc21f01a5 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Tue, 11 Nov 2025 18:05:06 +0000 Subject: [PATCH 017/110] Fix for max attempts. --- src/pipecat/services/speechmatics/tts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index b45c63ef7..76c7a1a8a 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -159,7 +159,7 @@ class SpeechmaticsTTSService(TTSService): attempt += 1 # Check if we've exceeded the maximum number of attempts - if attempt > self._params.max_retries: + if attempt >= self._params.max_retries: raise ValueError() # Report error frame From 8d21b54ef3e30cfdbc8fc15303740c04eb5dfbae Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Tue, 11 Nov 2025 18:24:08 +0000 Subject: [PATCH 018/110] Revert to `ErrorFrame`. --- src/pipecat/services/speechmatics/tts.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index 76c7a1a8a..b8fe172e7 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -16,7 +16,6 @@ from pydantic import BaseModel from pipecat.frames.frames import ( ErrorFrame, - FatalErrorFrame, Frame, TTSAudioRawFrame, TTSStartedFrame, @@ -174,15 +173,16 @@ class SpeechmaticsTTSService(TTSService): continue except (ValueError, ArithmeticError): - yield FatalErrorFrame( - error=f"{self} Service unavailable [503] (attempts {attempt})" + yield ErrorFrame( + error=f"{self} Service unavailable [503] (attempts {attempt})", + fatal=True, ) return # != 200 : Error if response.status != 200: - yield FatalErrorFrame( - error=f"{self} Service unavailable [{response.status}]" + yield ErrorFrame( + error=f"{self} Service unavailable [{response.status}]", fatal=True ) return @@ -225,7 +225,7 @@ class SpeechmaticsTTSService(TTSService): break except Exception as e: - yield ErrorFrame(error=f"{self}: Error generating TTS: {e}") + yield ErrorFrame(error=f"{self}: Error generating TTS: {e}", fatal=True) finally: # Emit the TTS stopped frame yield TTSStoppedFrame() From fff8aac18ceb2463794c05f24d0e335aeaac67e4 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 11 Nov 2025 17:25:40 -0300 Subject: [PATCH 019/110] Mention DeepgramFluxSTTService URL encode fix in changelog. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 974133011..4cca64597 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed issue where `DeepgramFluxSTTService` failed to connect if passing a + `keyterm` or `tag` containing a space. + - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. ## [0.0.94] - 2025-11-10 From 3c76917c1e4a0c1c5b71467fb9813e6bfd44ad9e Mon Sep 17 00:00:00 2001 From: Corvin Jaedicke Date: Wed, 12 Nov 2025 13:48:22 +0100 Subject: [PATCH 020/110] use async process function --- pyproject.toml | 80 ++++++++++++------------- src/pipecat/audio/filters/aic_filter.py | 2 +- uv.lock | 6 +- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a203b47b0..7b382f112 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,23 +1,23 @@ [build-system] -requires = ["setuptools>=64", "setuptools_scm>=8"] +requires = [ "setuptools>=64", "setuptools_scm>=8" ] build-backend = "setuptools.build_meta" [project] name = "pipecat-ai" -dynamic = ["version"] +dynamic = [ "version" ] description = "An open source framework for voice (and multimodal) assistants" license = "BSD-2-Clause" -license-files = ["LICENSE"] +license-files = [ "LICENSE" ] readme = "README.md" requires-python = ">=3.10" -keywords = ["webrtc", "audio", "video", "ai"] +keywords = [ "webrtc", "audio", "video", "ai" ] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Topic :: Communications :: Conferencing", "Topic :: Multimedia :: Sound/Audio", "Topic :: Multimedia :: Video", - "Topic :: Scientific/Engineering :: Artificial Intelligence" + "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ "aiofiles>=24.1.0,<25", @@ -45,30 +45,30 @@ Source = "https://github.com/pipecat-ai/pipecat" Website = "https://pipecat.ai" [project.optional-dependencies] -aic = [ "aic-sdk~=1.0.1" ] +aic = [ "aic-sdk~=1.1.0" ] anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] aws = [ "aioboto3~=15.0.0", "pipecat-ai[websockets-base]" ] aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.1.1; python_version>='3.12'" ] -azure = [ "azure-cognitiveservices-speech~=1.42.0"] +azure = [ "azure-cognitiveservices-speech~=1.42.0" ] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] -cerebras = [] -deepseek = [] +cerebras = [ ] +deepseek = [ ] daily = [ "daily-python~=0.21.0" ] deepgram = [ "deepgram-sdk~=4.7.0" ] elevenlabs = [ "pipecat-ai[websockets-base]" ] fal = [ "fal-client~=0.5.9" ] -fireworks = [] +fireworks = [ ] fish = [ "ormsgpack~=1.7.0", "pipecat-ai[websockets-base]" ] gladia = [ "pipecat-ai[websockets-base]" ] google = [ "google-cloud-speech>=2.33.0,<3", "google-cloud-texttospeech>=2.31.0,<3", "google-genai>=1.41.0,<2", "pipecat-ai[websockets-base]" ] -grok = [] +grok = [ ] groq = [ "groq~=0.23.0" ] gstreamer = [ "pygobject~=3.50.0" ] heygen = [ "livekit>=1.0.13", "pipecat-ai[websockets-base]" ] hume = [ "hume>=0.11.2" ] -inworld = [] +inworld = [ ] krisp = [ "pipecat-ai-krisp~=0.4.0" ] koala = [ "pvkoala~=2.0.3" ] langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-openai~=0.3.9" ] @@ -77,35 +77,35 @@ lmnt = [ "pipecat-ai[websockets-base]" ] local = [ "pyaudio~=0.2.14" ] mcp = [ "mcp[cli]>=1.11.0,<2" ] mem0 = [ "mem0ai~=0.1.94" ] -mistral = [] +mistral = [ ] mlx-whisper = [ "mlx-whisper~=0.4.2" ] moondream = [ "accelerate~=1.10.0", "einops~=0.8.0", "pyvips[binary]~=3.0.0", "timm~=1.0.13", "transformers>=4.48.0" ] -nim = [] +nim = [ ] neuphonic = [ "pipecat-ai[websockets-base]" ] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "pipecat-ai[websockets-base]" ] openpipe = [ "openpipe>=4.50.0,<6" ] -openrouter = [] -perplexity = [] +openrouter = [ ] +perplexity = [ ] playht = [ "pipecat-ai[websockets-base]" ] -qwen = [] +qwen = [ ] rime = [ "pipecat-ai[websockets-base]" ] riva = [ "nvidia-riva-client~=2.21.1" ] -runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.122.0", "pipecat-ai-small-webrtc-prebuilt>=1.0.0"] -sambanova = [] +runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.122.0", "pipecat-ai-small-webrtc-prebuilt>=1.0.0" ] +sambanova = [ ] sarvam = [ "sarvamai==0.1.21", "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk>=2.28.0,<3" ] local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "torchaudio>=2.5.0,<3" ] local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] -remote-smart-turn = [] +remote-smart-turn = [ ] silero = [ "onnxruntime>=1.20.1,<2" ] -simli = [ "simli-ai~=0.1.25"] +simli = [ "simli-ai~=0.1.25" ] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.1" ] speechmatics = [ "speechmatics-rt>=0.5.0" ] strands = [ "strands-agents>=1.9.1,<2" ] -tavus=[] -together = [] +tavus = [ ] +together = [ ] tracing = [ "opentelemetry-sdk>=1.33.0", "opentelemetry-api>=1.33.0", "opentelemetry-instrumentation>=0.54b0" ] ultravox = [ "transformers>=4.48.0", "vllm>=0.9.0" ] webrtc = [ "aiortc>=1.13.0,<2", "opencv-python>=4.11.0.86,<5" ] @@ -139,10 +139,10 @@ docs = [ ] [tool.setuptools.packages.find] -where = ["src"] +where = [ "src" ] [tool.setuptools.package-data] -"pipecat" = ["py.typed"] +"pipecat" = [ "py.typed" ] "pipecat.audio.dtmf" = [ "src/pipecat/audio/dtmf/dtmf-0.wav", "src/pipecat/audio/dtmf/dtmf-1.wav", @@ -157,13 +157,13 @@ where = ["src"] "src/pipecat/audio/dtmf/dtmf-pound.wav", "src/pipecat/audio/dtmf/dtmf-star.wav", ] -"pipecat.services.aws_nova_sonic" = ["src/pipecat/services/aws_nova_sonic/ready.wav"] -"pipecat.audio.turn.smart_turn.data" = ["src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx"] +"pipecat.services.aws_nova_sonic" = [ "src/pipecat/services/aws_nova_sonic/ready.wav" ] +"pipecat.audio.turn.smart_turn.data" = [ "src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx" ] [tool.pytest.ini_options] addopts = "--verbose" -testpaths = ["tests"] -pythonpath = ["src"] +testpaths = [ "tests" ] +pythonpath = [ "src" ] asyncio_default_fixture_loop_scope = "function" filterwarnings = [ "ignore:'audioop' is deprecated:DeprecationWarning", @@ -174,7 +174,7 @@ local_scheme = "no-local-version" fallback_version = "0.0.0-dev" [tool.ruff] -exclude = [".git", "*_pb2.py"] +exclude = [ ".git", "*_pb2.py" ] line-length = 100 [tool.ruff.lint] @@ -183,25 +183,25 @@ select = [ "I", # Import rules ] ignore = [ - "D105", # Missing docstring in magic methods (__str__, __repr__, etc.) + "D105", # Missing docstring in magic methods (__str__, __repr__, etc.) ] [tool.ruff.lint.per-file-ignores] # Skip docstring checks for non-source code -"examples/**/*.py" = ["D"] -"tests/**/*.py" = ["D"] -"scripts/**/*.py" = ["D"] -"docs/**/*.py" = ["D"] +"examples/**/*.py" = [ "D" ] +"tests/**/*.py" = [ "D" ] +"scripts/**/*.py" = [ "D" ] +"docs/**/*.py" = [ "D" ] # Skip D104 (missing docstring in public package) for __init__.py files -"**/__init__.py" = ["D104"] +"**/__init__.py" = [ "D104" ] # Skip specific rules for generated protobuf files -"**/*_pb2.py" = ["D"] -"src/pipecat/services/__init__.py" = ["D"] +"**/*_pb2.py" = [ "D" ] +"src/pipecat/services/__init__.py" = [ "D" ] [tool.ruff.lint.pydocstyle] convention = "google" [tool.coverage.run] command_line = "--module pytest" -source = ["src"] -omit = ["*/tests/*"] +source = [ "src" ] +omit = [ "*/tests/*" ] diff --git a/src/pipecat/audio/filters/aic_filter.py b/src/pipecat/audio/filters/aic_filter.py index 8c2f3e7f4..3f1af2f57 100644 --- a/src/pipecat/audio/filters/aic_filter.py +++ b/src/pipecat/audio/filters/aic_filter.py @@ -185,7 +185,7 @@ class AICFilter(BaseAudioFilter): ) # Process planar in-place; returns ndarray (same shape) - out_f32 = self._aic.process(block_f32) + out_f32 = await self._aic.process_async(block_f32) # Convert back to int16 bytes, planar layout out_i16 = np.clip(out_f32 * 32768.0, -32768, 32767).astype(np.int16) diff --git a/uv.lock b/uv.lock index 0dc6c74f8..eabe03714 100644 --- a/uv.lock +++ b/uv.lock @@ -36,12 +36,12 @@ wheels = [ [[package]] name = "aic-sdk" -version = "1.0.2" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/90/b02e853e863c303f8456c689b42ac24ad403b781adc9642d0a91ed4bed7e/aic_sdk-1.0.2.tar.gz", hash = "sha256:239097dd3aaa8a8a0fd7542b75d2510cb34144caec796370639b7c636acbc56e", size = 32059, upload-time = "2025-08-24T09:20:03.9Z" } +sdist = { url = "https://files.pythonhosted.org/packages/99/83/bf38b95d98c67b8ebc574fb4a4f23c07a3740b51992d7522976173d30b98/aic_sdk-1.1.0.tar.gz", hash = "sha256:04e08df695581c8cb4db8acca20e73815e9f449e7bd08e0162fd55518c727963", size = 34954, upload-time = "2025-11-11T20:45:24.25Z" } [[package]] name = "aioboto3" @@ -4647,7 +4647,7 @@ docs = [ [package.metadata] requires-dist = [ { name = "accelerate", marker = "extra == 'moondream'", specifier = "~=1.10.0" }, - { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.0.1" }, + { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.1.0" }, { name = "aioboto3", marker = "extra == 'aws'", specifier = "~=15.0.0" }, { name = "aiofiles", specifier = ">=24.1.0,<25" }, { name = "aiohttp", specifier = ">=3.11.12,<4" }, From 2006a64def030859c2b043141fb513d3fc14a65b Mon Sep 17 00:00:00 2001 From: James Hush Date: Wed, 12 Nov 2025 14:58:00 +0100 Subject: [PATCH 021/110] Fix Langfuse tracing for GoogleLLMService with universal LLMContext (#3025) * Fix Langfuse tracing for GoogleLLMService with universal LLMContext - Fixed issue where input appeared as null in Langfuse dashboard for GoogleLLMService - Added fallback to use adapter's get_messages_for_logging() for universal LLMContext - Ensures proper message format conversion for Google/Gemini services - Handles system message conversion to system_instruction format - Also fixes serialization of empty message lists ([] now serializes correctly) This fix ensures Langfuse tracing works correctly for Google services using both OpenAILLMContext/GoogleLLMContext and the universal LLMContext. * Add unit tests for Langfuse tracing with GoogleLLMService - Test that tracing correctly captures messages with universal LLMContext - Test that empty message lists are properly serialized - Test that adapter's get_messages_for_logging is used instead of context method - All tests verify that input is correctly added to Langfuse spans * Fix test mocking to patch opentelemetry.trace.get_tracer correctly The tests were failing in CI because they were trying to patch 'pipecat.utils.tracing.service_decorators.trace' which doesn't exist as an attribute. The trace module is imported from opentelemetry, so we need to patch 'opentelemetry.trace.get_tracer' instead. * Skip tracing tests when opentelemetry is not installed The tracing dependencies (opentelemetry) are optional in Pipecat and not installed in the CI environment. Added a skipif marker to skip these tests when opentelemetry is not available, preventing CI failures while still allowing the tests to run when tracing dependencies are installed locally. * Install tracing dependencies in GitHub Actions CI Instead of skipping the tracing tests, install the 'tracing' extra (opentelemetry) in the CI environment so the tests can run properly. Removed the skipif condition from the tests since opentelemetry will now be available in CI. * Use the context type to determine which messages to use, fix tool_count and tools (#3032) --------- Co-authored-by: Mark Backman --- CHANGELOG.md | 5 +- .../utils/tracing/service_decorators.py | 70 ++++++++++++------- 2 files changed, 47 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cca64597..60bfa354b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,13 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## [Unreleased] ### Fixed +- Fixed an issue with OpenTelemetry where tracing wasn't correctly displaying + LLM completions and tools when using the universal `LLMContext`. + - Fixed issue where `DeepgramFluxSTTService` failed to connect if passing a `keyterm` or `tag` containing a space. diff --git a/src/pipecat/utils/tracing/service_decorators.py b/src/pipecat/utils/tracing/service_decorators.py index 3935a4afc..3c743c1a6 100644 --- a/src/pipecat/utils/tracing/service_decorators.py +++ b/src/pipecat/utils/tracing/service_decorators.py @@ -23,6 +23,8 @@ if TYPE_CHECKING: from opentelemetry import context as context_api from opentelemetry import trace +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.utils.tracing.service_attributes import ( add_gemini_live_span_attributes, add_llm_span_attributes, @@ -382,43 +384,57 @@ def traced_llm(func: Optional[Callable] = None, *, name: Optional[str] = None) - # Replace push_frame to capture output self.push_frame = traced_push_frame - # Detect if we're using Google's service - is_google_service = "google" in service_class_name.lower() - - # Try to get messages based on service type + # Get messages for logging + # For OpenAILLMContext: use context's own get_messages_for_logging() method + # For LLMContext: use adapter's get_messages_for_logging() which returns + # messages in provider's native format with sensitive data sanitized messages = None serialized_messages = None - # TODO: Revisit once we unify the messages across services - if is_google_service: - # Handle Google service specifically - if hasattr(context, "get_messages_for_logging"): - messages = context.get_messages_for_logging() - else: - # Handle other services like OpenAI - if hasattr(context, "get_messages"): - messages = context.get_messages() - elif hasattr(context, "messages"): - messages = context.messages + if isinstance(context, OpenAILLMContext): + # OpenAILLMContext and subclasses have their own method + messages = context.get_messages_for_logging() + elif isinstance(context, LLMContext): + # Universal LLMContext - use adapter for provider-native format + if hasattr(self, "get_llm_adapter"): + adapter = self.get_llm_adapter() + messages = adapter.get_messages_for_logging(context) + elif hasattr(context, "get_messages"): + # Fallback for unknown context types + messages = context.get_messages() + elif hasattr(context, "messages"): + messages = context.messages # Serialize messages if available if messages: - try: - serialized_messages = json.dumps(messages) - except Exception as e: - serialized_messages = f"Error serializing messages: {str(e)}" + serialized_messages = json.dumps(messages) - # Get tools, system message, etc. based on the service type - tools = getattr(context, "tools", None) + # Get tools + # For OpenAILLMContext: tools may need adapter conversion if set + # For LLMContext: use adapter's from_standard_tools() to convert ToolsSchema + tools = None serialized_tools = None tool_count = 0 - if tools: - try: - serialized_tools = json.dumps(tools) - tool_count = len(tools) if isinstance(tools, list) else 1 - except Exception as e: - serialized_tools = f"Error serializing tools: {str(e)}" + if isinstance(context, OpenAILLMContext): + # OpenAILLMContext: tools property handles adapter conversion internally + tools = context.tools + elif isinstance(context, LLMContext): + # Universal LLMContext - use adapter to convert ToolsSchema + if hasattr(self, "get_llm_adapter") and hasattr(context, "tools"): + adapter = self.get_llm_adapter() + tools = adapter.from_standard_tools(context.tools) + elif hasattr(context, "tools"): + # Fallback for unknown context types + tools = context.tools + + # Serialize and count tools if available + # Check if tools is not None and not NOT_GIVEN (using attribute check as fallback) + if tools is not None and not ( + hasattr(tools, "__name__") and tools.__name__ == "NOT_GIVEN" + ): + serialized_tools = json.dumps(tools) + tool_count = len(tools) if isinstance(tools, list) else 1 # Handle system message for different services system_message = None From 5222ff99deda0752dbf7db0f807ae49157275ef4 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 12 Nov 2025 10:02:50 -0500 Subject: [PATCH 022/110] Apply `includes_inter_frame_spaces = True` in all LLM and TTS services that need it. Note that for `LLMTextFrame`s, the right behavior is pretty much always `includes_inter_frame_spaces = True`. I decided *not* to go ahead and make that the default for `LLMTextFrame`s, though, simply to not introduce a subtle behavior change for creative/unexpected use-cases that were relying on text in hand-crafted `LLMTextFrame`s being handled a certain way. Ditto for `TTSTextFrame`s. Also, fix an issue in `NeuphonicTTSService` where it wasn't pushing `TTSTextFrame`s. Also, fix the broken `SarvamHttpTTSService` example. Also, add a couple of missing examples. --- CHANGELOG.md | 12 ++ .../07f-interruptible-azure-http.py | 135 +++++++++++++++++ .../07n-interruptible-google-http.py | 139 ++++++++++++++++++ .../07z-interruptible-sarvam-http.py | 3 +- src/pipecat/services/anthropic/llm.py | 4 +- src/pipecat/services/asyncai/tts.py | 18 +++ src/pipecat/services/aws/llm.py | 4 +- src/pipecat/services/aws/nova_sonic/llm.py | 8 +- src/pipecat/services/aws/tts.py | 9 ++ src/pipecat/services/azure/tts.py | 9 ++ src/pipecat/services/deepgram/tts.py | 18 +++ src/pipecat/services/fish/tts.py | 9 ++ src/pipecat/services/google/llm.py | 4 +- src/pipecat/services/google/tts.py | 18 +++ src/pipecat/services/groq/tts.py | 9 ++ src/pipecat/services/hume/tts.py | 9 ++ src/pipecat/services/inworld/tts.py | 9 ++ src/pipecat/services/lmnt/tts.py | 9 ++ src/pipecat/services/minimax/tts.py | 9 ++ src/pipecat/services/neuphonic/tts.py | 19 ++- src/pipecat/services/openai/base_llm.py | 4 +- src/pipecat/services/piper/tts.py | 9 ++ src/pipecat/services/rime/tts.py | 9 ++ src/pipecat/services/riva/tts.py | 9 ++ src/pipecat/services/sambanova/llm.py | 4 +- src/pipecat/services/sarvam/tts.py | 18 +++ src/pipecat/services/speechmatics/tts.py | 9 ++ src/pipecat/services/tts_service.py | 21 ++- 28 files changed, 527 insertions(+), 10 deletions(-) create mode 100644 examples/foundational/07f-interruptible-azure-http.py create mode 100644 examples/foundational/07n-interruptible-google-http.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 74506687d..a43553f0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added a `TTSService.includes_inter_frame_spaces` property getter, so that TTS + services that subclass `TTSService` can indicate whether the text in the + `TTSTextFrame`s they push already contain any necessary inter-frame spaces. + ### Fixed +- Fixed subtle issue of assistant context messages ending up with double spaces + between words or sentences. + +- Fixed an issue where `NeuphonicTTSService` wasn't pushing `TTSTextFrame`s, + meaning assistant messages weren't being written to context. + - Fixed an issue with OpenTelemetry where tracing wasn't correctly displaying LLM completions and tools when using the universal `LLMContext`. diff --git a/examples/foundational/07f-interruptible-azure-http.py b/examples/foundational/07f-interruptible-azure-http.py new file mode 100644 index 000000000..63971f156 --- /dev/null +++ b/examples/foundational/07f-interruptible-azure-http.py @@ -0,0 +1,135 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.azure.llm import AzureLLMService +from pipecat.services.azure.stt import AzureSTTService +from pipecat.services.azure.tts import AzureHttpTTSService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = AzureSTTService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION"), + ) + + tts = AzureHttpTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION"), + ) + + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL"), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/07n-interruptible-google-http.py b/examples/foundational/07n-interruptible-google-http.py new file mode 100644 index 000000000..136f09d7f --- /dev/null +++ b/examples/foundational/07n-interruptible-google-http.py @@ -0,0 +1,139 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.google.llm import GoogleLLMService +from pipecat.services.google.stt import GoogleSTTService +from pipecat.services.google.tts import GoogleHttpTTSService, GoogleTTSService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = GoogleSTTService( + params=GoogleSTTService.InputParams(languages=Language.EN_US, model="chirp_3"), + credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + location="us", + ) + + tts = GoogleHttpTTSService( + voice_id="en-US-Chirp3-HD-Charon", + params=GoogleHttpTTSService.InputParams(language=Language.EN_US), + credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + ) + + llm = GoogleLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + model="gemini-2.5-flash", + # turn on thinking if you want it + # params=GoogleLLMService.InputParams(extra={"thinking_config": {"thinking_budget": 4096}}),) + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User respones + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/07z-interruptible-sarvam-http.py b/examples/foundational/07z-interruptible-sarvam-http.py index 0821167ef..4d06affe5 100644 --- a/examples/foundational/07z-interruptible-sarvam-http.py +++ b/examples/foundational/07z-interruptible-sarvam-http.py @@ -15,6 +15,7 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -112,7 +113,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Client connected") # Kick off the conversation. messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index 2e3e0272d..a5c9ee791 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -373,7 +373,9 @@ class AnthropicLLMService(LLMService): if event.type == "content_block_delta": if hasattr(event.delta, "text"): - await self.push_frame(LLMTextFrame(event.delta.text)) + frame = LLMTextFrame(event.delta.text) + frame.includes_inter_frame_spaces = True + await self.push_frame(frame) completion_tokens_estimate += self._estimate_tokens(event.delta.text) elif hasattr(event.delta, "partial_json") and tool_use_block: json_accumulator += event.delta.partial_json diff --git a/src/pipecat/services/asyncai/tts.py b/src/pipecat/services/asyncai/tts.py index fe067e6b1..78cdd7ef8 100644 --- a/src/pipecat/services/asyncai/tts.py +++ b/src/pipecat/services/asyncai/tts.py @@ -146,6 +146,15 @@ class AsyncAITTSService(InterruptibleTTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that AsyncAI TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that AsyncAI's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Async language format. @@ -420,6 +429,15 @@ class AsyncAIHttpTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that AsyncAI TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that AsyncAI's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Async language format. diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index ccbac43b7..147a8c12a 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -1078,7 +1078,9 @@ class AWSBedrockLLMService(LLMService): if "contentBlockDelta" in event: delta = event["contentBlockDelta"]["delta"] if "text" in delta: - await self.push_frame(LLMTextFrame(delta["text"])) + frame = LLMTextFrame(delta["text"]) + frame.includes_inter_frame_spaces = True + await self.push_frame(frame) completion_tokens_estimate += self._estimate_tokens(delta["text"]) elif "toolUse" in delta and "input" in delta["toolUse"]: # Handle partial JSON for tool use diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 4ad9d05ea..2572b03cb 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -1027,7 +1027,9 @@ class AWSNovaSonicLLMService(LLMService): logger.debug(f"Assistant response text added: {text}") # Report the text of the assistant response. - await self.push_frame(TTSTextFrame(text)) + frame = TTSTextFrame(text) + frame.includes_inter_frame_spaces = True + await self.push_frame(frame) # HACK: here we're also buffering the assistant text ourselves as a # backup rather than relying solely on the assistant context aggregator @@ -1060,7 +1062,9 @@ class AWSNovaSonicLLMService(LLMService): # TTSTextFrame would be ignored otherwise (the interruption frame # would have cleared the assistant aggregator state). await self.push_frame(LLMFullResponseStartFrame()) - await self.push_frame(TTSTextFrame(self._assistant_text_buffer)) + frame = TTSTextFrame(self._assistant_text_buffer) + frame.includes_inter_frame_spaces = True + await self.push_frame(frame) self._may_need_repush_assistant_text = False # Report the end of the assistant response. diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index f22c42399..cbc35b123 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -209,6 +209,15 @@ class AWSPollyTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that AWS TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that AWS's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to AWS Polly language format. diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index 15b4f1256..d0ae42796 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -151,6 +151,15 @@ class AzureBaseTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Azure TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Azure's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Azure language format. diff --git a/src/pipecat/services/deepgram/tts.py b/src/pipecat/services/deepgram/tts.py index f3869c0ba..2c816e4a9 100644 --- a/src/pipecat/services/deepgram/tts.py +++ b/src/pipecat/services/deepgram/tts.py @@ -79,6 +79,15 @@ class DeepgramTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Deepgram TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Deepgram's text frames include necessary inter-frame spaces. + """ + return True + @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Deepgram's TTS API. @@ -168,6 +177,15 @@ class DeepgramHttpTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Deepgram TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Deepgram's text frames include necessary inter-frame spaces. + """ + return True + @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Deepgram's TTS API. diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index 669d2ce97..1abe6aca1 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -159,6 +159,15 @@ class FishAudioTTSService(InterruptibleTTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Fish Audio TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Fish Audio's text frames include necessary inter-frame spaces. + """ + return True + async def set_model(self, model: str): """Set the TTS model and reconnect. diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 883932b76..ad5fd70a7 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -920,7 +920,9 @@ class GoogleLLMService(LLMService): for part in candidate.content.parts: if not part.thought and part.text: search_result += part.text - await self.push_frame(LLMTextFrame(part.text)) + frame = LLMTextFrame(part.text) + frame.includes_inter_frame_spaces = True + await self.push_frame(frame) elif part.function_call: function_call = part.function_call id = function_call.id or str(uuid.uuid4()) diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index bfbbd8a3c..bd3dbc203 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -606,6 +606,15 @@ class GoogleTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Google TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Google's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Google TTS language format. @@ -840,6 +849,15 @@ class GeminiTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Gemini TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Gemini's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Gemini TTS language format. diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index 68ba4a598..6bd49d1a3 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -105,6 +105,15 @@ class GroqTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Groq TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Groq's text frames include necessary inter-frame spaces. + """ + return True + @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Groq's TTS API. diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 34947fb44..f8b2bbf27 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -110,6 +110,15 @@ class HumeTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Hume TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Hume's text frames include necessary inter-frame spaces. + """ + return True + async def start(self, frame: StartFrame) -> None: """Start the service. diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index eef1440e3..9bb939518 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -250,6 +250,15 @@ class InworldTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Inworld TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Inworld's text frames include necessary inter-frame spaces. + """ + return True + async def start(self, frame: StartFrame): """Start the Inworld TTS service. diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index f71e2a186..538c1ef93 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -124,6 +124,15 @@ class LmntTTSService(InterruptibleTTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that LMNT TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that LMNT's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to LMNT service language format. diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index c1a8abb99..c0a6b6aaa 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -194,6 +194,15 @@ class MiniMaxHttpTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that MiniMax TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that MiniMax's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to MiniMax service language format. diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index 3449dea0c..22a6e9999 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -117,7 +117,6 @@ class NeuphonicTTSService(InterruptibleTTSService): """ super().__init__( aggregate_sentences=aggregate_sentences, - push_text_frames=False, push_stop_frames=True, stop_frame_timeout_s=2.0, sample_rate=sample_rate, @@ -152,6 +151,15 @@ class NeuphonicTTSService(InterruptibleTTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Neuphonic TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Neuphonic's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Neuphonic service language format. @@ -437,6 +445,15 @@ class NeuphonicHttpTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Neuphonic TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Neuphonic's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Neuphonic service language format. diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index d020e1106..5a8c1ab31 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -390,7 +390,9 @@ class BaseOpenAILLMService(LLMService): # Keep iterating through the response to collect all the argument fragments arguments += tool_call.function.arguments elif chunk.choices[0].delta.content: - await self.push_frame(LLMTextFrame(chunk.choices[0].delta.content)) + frame = LLMTextFrame(chunk.choices[0].delta.content) + frame.includes_inter_frame_spaces = True + await self.push_frame(frame) # When gpt-4o-audio / gpt-4o-mini-audio is used for llm or stt+llm # we need to get LLMTextFrame for the transcript diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index fa43a720c..73addb9d1 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -66,6 +66,15 @@ class PiperTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Piper TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Piper's text frames include necessary inter-frame spaces. + """ + return True + @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Piper's HTTP API. diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index f0dd6b297..0ac37c471 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -496,6 +496,15 @@ class RimeHttpTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Rime TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Rime's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> str | None: """Convert pipecat language to Rime language code. diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py index 3554c5558..d051965ed 100644 --- a/src/pipecat/services/riva/tts.py +++ b/src/pipecat/services/riva/tts.py @@ -112,6 +112,15 @@ class RivaTTSService(TTSService): riva.client.proto.riva_tts_pb2.RivaSynthesisConfigRequest() ) + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Riva TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Riva's text frames include necessary inter-frame spaces. + """ + return True + async def set_model(self, model: str): """Attempt to set the TTS model. diff --git a/src/pipecat/services/sambanova/llm.py b/src/pipecat/services/sambanova/llm.py index 5ed600457..76f11e81c 100644 --- a/src/pipecat/services/sambanova/llm.py +++ b/src/pipecat/services/sambanova/llm.py @@ -176,7 +176,9 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore # Keep iterating through the response to collect all the argument fragments arguments += tool_call.function.arguments elif chunk.choices[0].delta.content: - await self.push_frame(LLMTextFrame(chunk.choices[0].delta.content)) + frame = LLMTextFrame(chunk.choices[0].delta.content) + frame.includes_inter_frame_spaces = True + await self.push_frame(frame) # When gpt-4o-audio / gpt-4o-mini-audio is used for llm or stt+llm # we need to get LLMTextFrame for the transcript diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index e8582227a..9ff037938 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -195,6 +195,15 @@ class SarvamHttpTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Sarvam TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Sarvam's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Sarvam AI language format. @@ -458,6 +467,15 @@ class SarvamTTSService(InterruptibleTTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Sarvam TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Sarvam's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Sarvam AI language format. diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index b8fe172e7..e115d5a7c 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -105,6 +105,15 @@ class SpeechmaticsTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Speechmatics TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Speechmatics's text frames include necessary inter-frame spaces. + """ + return True + @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Speechmatics' HTTP API. diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index b356c7244..29c54f497 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -192,6 +192,23 @@ class TTSService(AIService): CHUNK_SECONDS = 0.5 return int(self.sample_rate * CHUNK_SECONDS * 2) # 2 bytes/sample + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates whether TTSTextFrames include necesary inter-frame spaces. + + When True, the TTSTextFrame objects pushed by this service already + include all necessary spaces between subsequent frames. When False, + downstream processors (like the assistant context aggregator) may need + to add spacing. + + Subclasses should override this property to return True if their text + generation process already includes necessary inter-frame spaces. + + Returns: + False by default. Subclasses can override to return True. + """ + return False + async def set_model(self, model: str): """Set the TTS model to use. @@ -490,7 +507,9 @@ class TTSService(AIService): if self._push_text_frames: # We send the original text after the audio. This way, if we are # interrupted, the text is not added to the assistant context. - await self.push_frame(TTSTextFrame(text)) + frame = TTSTextFrame(text) + frame.includes_inter_frame_spaces = self.includes_inter_frame_spaces + await self.push_frame(frame) async def _stop_frame_handler(self): has_started = False From 1ad6405ebb33d651043eb6f73ab09d76d0f89771 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 12 Nov 2025 17:07:43 -0500 Subject: [PATCH 023/110] Override `includes_inter_frame_spaces` in: - `GoogleHttpTTSService` - `OpenAITTSService` The reason I skipped this work in an earlier PR was because these services seemed to be emitting long, punctuation-free text frames. It turns out that the issue was with the LLM prompt, though, resulting in the LLM nondeterministically excluding all punctuation. An upcoming commit will address that prompt issue. --- src/pipecat/services/google/tts.py | 9 +++++++++ src/pipecat/services/openai/tts.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index bd3dbc203..5575d7d44 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -352,6 +352,15 @@ class GoogleHttpTTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Google TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Google's text frames include necessary inter-frame spaces. + """ + return True + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Google TTS language format. diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index cdf0d11ac..af580e4a0 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -131,6 +131,15 @@ class OpenAITTSService(TTSService): """ return True + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that OpenAI TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that OpenAI's text frames include necessary inter-frame spaces. + """ + return True + async def set_model(self, model: str): """Set the TTS model to use. From 1802f949ef1d103343c68231d1deab73daec286c Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 12 Nov 2025 17:12:03 -0500 Subject: [PATCH 024/110] Fix an issue with some examples where punctuation was missing from the LLM output, by tweaking the LLM prompt. --- examples/foundational/04-transports-small-webrtc.py | 2 +- examples/foundational/04a-transports-daily.py | 2 +- examples/foundational/04b-transports-livekit.py | 2 +- examples/foundational/06-listen-and-respond.py | 2 +- examples/foundational/06a-image-sync.py | 2 +- examples/foundational/07-interruptible-cartesia-http.py | 2 +- examples/foundational/07-interruptible.py | 2 +- examples/foundational/07a-interruptible-speechmatics-vad.py | 2 +- examples/foundational/07a-interruptible-speechmatics.py | 2 +- examples/foundational/07aa-interruptible-soniox.py | 2 +- examples/foundational/07ab-interruptible-inworld-http.py | 2 +- examples/foundational/07ac-interruptible-asyncai-http.py | 2 +- examples/foundational/07ac-interruptible-asyncai.py | 2 +- examples/foundational/07ad-interruptible-aicoustics.py | 2 +- examples/foundational/07ae-interruptible-hume.py | 2 +- examples/foundational/07c-interruptible-deepgram-flux.py | 2 +- examples/foundational/07c-interruptible-deepgram-http.py | 2 +- examples/foundational/07c-interruptible-deepgram-vad.py | 2 +- examples/foundational/07c-interruptible-deepgram.py | 2 +- examples/foundational/07d-interruptible-elevenlabs-http.py | 2 +- examples/foundational/07d-interruptible-elevenlabs.py | 2 +- examples/foundational/07e-interruptible-playht-http.py | 2 +- examples/foundational/07e-interruptible-playht.py | 2 +- examples/foundational/07f-interruptible-azure-http.py | 2 +- examples/foundational/07f-interruptible-azure.py | 2 +- examples/foundational/07g-interruptible-openai.py | 2 +- examples/foundational/07h-interruptible-openpipe.py | 2 +- examples/foundational/07i-interruptible-xtts.py | 2 +- examples/foundational/07j-interruptible-gladia.py | 2 +- examples/foundational/07k-interruptible-lmnt.py | 2 +- examples/foundational/07l-interruptible-groq.py | 2 +- examples/foundational/07m-interruptible-aws.py | 2 +- examples/foundational/07n-interruptible-gemini-image.py | 2 +- examples/foundational/07n-interruptible-google-http.py | 2 +- examples/foundational/07n-interruptible-google.py | 2 +- examples/foundational/07o-interruptible-assemblyai.py | 2 +- examples/foundational/07p-interruptible-krisp-viva.py | 2 +- examples/foundational/07p-interruptible-krisp.py | 2 +- examples/foundational/07q-interruptible-rime-http.py | 2 +- examples/foundational/07q-interruptible-rime.py | 2 +- examples/foundational/07r-interruptible-riva-nim.py | 2 +- examples/foundational/07s-interruptible-google-audio-in.py | 2 +- examples/foundational/07t-interruptible-fish.py | 2 +- examples/foundational/07v-interruptible-neuphonic-http.py | 2 +- examples/foundational/07v-interruptible-neuphonic.py | 2 +- examples/foundational/07w-interruptible-fal.py | 2 +- examples/foundational/07x-interruptible-local.py | 2 +- examples/foundational/07y-interruptible-minimax.py | 2 +- examples/foundational/07z-interruptible-sarvam-http.py | 2 +- examples/foundational/07z-interruptible-sarvam.py | 2 +- examples/foundational/08-custom-frame-processor.py | 2 +- examples/foundational/12-describe-image-openai.py | 2 +- examples/foundational/12a-describe-image-anthropic.py | 2 +- examples/foundational/12b-describe-image-aws.py | 2 +- examples/foundational/12c-describe-image-gemini-flash.py | 2 +- examples/foundational/14-function-calling.py | 2 +- examples/foundational/14c-function-calling-together.py | 2 +- examples/foundational/14d-function-calling-anthropic-video.py | 2 +- examples/foundational/14d-function-calling-aws-video.py | 2 +- .../foundational/14d-function-calling-gemini-flash-video.py | 2 +- examples/foundational/14d-function-calling-moondream-video.py | 2 +- examples/foundational/14d-function-calling-openai-video.py | 2 +- examples/foundational/14f-function-calling-groq.py | 2 +- examples/foundational/14g-function-calling-grok.py | 2 +- examples/foundational/14h-function-calling-azure.py | 2 +- examples/foundational/14i-function-calling-fireworks.py | 2 +- examples/foundational/14j-function-calling-nim.py | 2 +- examples/foundational/14m-function-calling-openrouter.py | 2 +- examples/foundational/14n-function-calling-perplexity.py | 2 +- examples/foundational/14q-function-calling-qwen.py | 2 +- examples/foundational/14r-function-calling-aws.py | 2 +- examples/foundational/14s-function-calling-sambanova.py | 2 +- examples/foundational/14t-function-calling-direct.py | 2 +- examples/foundational/14u-function-calling-ollama.py | 2 +- examples/foundational/14v-function-calling-openai.py | 2 +- examples/foundational/14w-function-calling-mistral.py | 2 +- examples/foundational/14x-function-calling-openpipe.py | 2 +- examples/foundational/16-gpu-container-local-bot.py | 2 +- examples/foundational/17-detect-user-idle.py | 2 +- examples/foundational/20a-persistent-context-openai.py | 2 +- examples/foundational/20c-persistent-context-anthropic.py | 2 +- examples/foundational/21-tavus-transport.py | 2 +- examples/foundational/21a-tavus-video-service.py | 2 +- examples/foundational/22-natural-conversation.py | 2 +- examples/foundational/22b-natural-conversation-proposal.py | 2 +- examples/foundational/22c-natural-conversation-mixed-llms.py | 2 +- .../foundational/22d-natural-conversation-gemini-audio.py | 2 +- examples/foundational/23-bot-background-sound.py | 2 +- examples/foundational/25-google-audio-in.py | 2 +- examples/foundational/26-gemini-live.py | 2 +- examples/foundational/26d-gemini-live-text.py | 2 +- examples/foundational/26f-gemini-live-files-api.py | 2 +- examples/foundational/26g-gemini-live-groundingMetadata.py | 2 +- examples/foundational/27-simli-layer.py | 2 +- examples/foundational/28-transcription-processor.py | 2 +- examples/foundational/29-turn-tracking-observer.py | 2 +- examples/foundational/30-observer.py | 2 +- examples/foundational/36-user-email-gathering.py | 4 ++-- examples/foundational/38-smart-turn-fal.py | 2 +- examples/foundational/38a-smart-turn-local-coreml.py | 2 +- examples/foundational/38b-smart-turn-local.py | 2 +- examples/foundational/39-mcp-stdio.py | 2 +- examples/foundational/39a-mcp-run-sse.py | 2 +- examples/foundational/39b-multiple-mcp.py | 2 +- examples/foundational/39c-mcp-run-http.py | 2 +- examples/foundational/39d-mcp-run-http-gemini-live.py | 2 +- examples/foundational/42-interruption-config.py | 2 +- examples/foundational/43-heygen-transport.py | 2 +- examples/foundational/43a-heygen-video-service.py | 2 +- examples/foundational/44-voicemail-detection.py | 2 +- examples/foundational/45-before-and-after-events.py | 2 +- examples/foundational/46-video-processing.py | 2 +- examples/foundational/47-sentry-metrics.py | 2 +- examples/foundational/48-service-switcher.py | 2 +- 114 files changed, 115 insertions(+), 115 deletions(-) diff --git a/examples/foundational/04-transports-small-webrtc.py b/examples/foundational/04-transports-small-webrtc.py index 997b917f9..292d5f961 100644 --- a/examples/foundational/04-transports-small-webrtc.py +++ b/examples/foundational/04-transports-small-webrtc.py @@ -77,7 +77,7 @@ async def run_example(webrtc_connection: SmallWebRTCConnection): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/04a-transports-daily.py b/examples/foundational/04a-transports-daily.py index 50567e84b..228905e89 100644 --- a/examples/foundational/04a-transports-daily.py +++ b/examples/foundational/04a-transports-daily.py @@ -60,7 +60,7 @@ async def main(): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/04b-transports-livekit.py b/examples/foundational/04b-transports-livekit.py index 402fd79ca..51705240f 100644 --- a/examples/foundational/04b-transports-livekit.py +++ b/examples/foundational/04b-transports-livekit.py @@ -69,7 +69,7 @@ async def main(): "role": "system", "content": "You are a helpful LLM in a WebRTC call. " "Your goal is to demonstrate your capabilities in a succinct way. " - "Your output will be converted to audio so don't include special characters in your answers. " + "Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. " "Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index 92e04e25a..638574975 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -100,7 +100,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index 4f8816df6..b4ea804eb 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -113,7 +113,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07-interruptible-cartesia-http.py b/examples/foundational/07-interruptible-cartesia-http.py index 569443a79..599cbecc5 100644 --- a/examples/foundational/07-interruptible-cartesia-http.py +++ b/examples/foundational/07-interruptible-cartesia-http.py @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 81ba692c7..13a4738fc 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -70,7 +70,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07a-interruptible-speechmatics-vad.py b/examples/foundational/07a-interruptible-speechmatics-vad.py index 6e78a5147..2c42a3e4a 100644 --- a/examples/foundational/07a-interruptible-speechmatics-vad.py +++ b/examples/foundational/07a-interruptible-speechmatics-vad.py @@ -121,7 +121,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): "content": ( "You are a helpful British assistant called Sarah. " "Your goal is to demonstrate your capabilities in a succinct way. " - "Your output will be converted to audio so don't include special characters in your answers. " + "Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. " "Always include punctuation in your responses. " "Give very short replies - do not give longer replies unless strictly necessary. " "Respond to what the user said in a concise, funny, creative and helpful way. " diff --git a/examples/foundational/07a-interruptible-speechmatics.py b/examples/foundational/07a-interruptible-speechmatics.py index 36ac39b82..4edb62f7e 100644 --- a/examples/foundational/07a-interruptible-speechmatics.py +++ b/examples/foundational/07a-interruptible-speechmatics.py @@ -111,7 +111,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): "content": ( "You are a helpful British assistant called Sarah. " "Your goal is to demonstrate your capabilities in a succinct way. " - "Your output will be converted to audio so don't include special characters in your answers. " + "Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. " "Always include punctuation in your responses. " "Give very short replies - do not give longer replies unless strictly necessary. " "Respond to what the user said in a concise, funny, creative and helpful way. " diff --git a/examples/foundational/07aa-interruptible-soniox.py b/examples/foundational/07aa-interruptible-soniox.py index addfde49a..a5462a4af 100644 --- a/examples/foundational/07aa-interruptible-soniox.py +++ b/examples/foundational/07aa-interruptible-soniox.py @@ -70,7 +70,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07ab-interruptible-inworld-http.py b/examples/foundational/07ab-interruptible-inworld-http.py index 907a358d2..d86f1b241 100644 --- a/examples/foundational/07ab-interruptible-inworld-http.py +++ b/examples/foundational/07ab-interruptible-inworld-http.py @@ -81,7 +81,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are very knowledgable about dogs. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are very knowledgable about dogs. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07ac-interruptible-asyncai-http.py b/examples/foundational/07ac-interruptible-asyncai-http.py index 7a92f0f7c..da2df63b0 100644 --- a/examples/foundational/07ac-interruptible-asyncai-http.py +++ b/examples/foundational/07ac-interruptible-asyncai-http.py @@ -76,7 +76,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07ac-interruptible-asyncai.py b/examples/foundational/07ac-interruptible-asyncai.py index 8216c317b..08680e569 100644 --- a/examples/foundational/07ac-interruptible-asyncai.py +++ b/examples/foundational/07ac-interruptible-asyncai.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07ad-interruptible-aicoustics.py b/examples/foundational/07ad-interruptible-aicoustics.py index aa647ff33..4b5fea5ae 100644 --- a/examples/foundational/07ad-interruptible-aicoustics.py +++ b/examples/foundational/07ad-interruptible-aicoustics.py @@ -95,7 +95,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index dedc0450d..2c7a651dc 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07c-interruptible-deepgram-flux.py b/examples/foundational/07c-interruptible-deepgram-flux.py index 75a022a5c..39fa8fc6a 100644 --- a/examples/foundational/07c-interruptible-deepgram-flux.py +++ b/examples/foundational/07c-interruptible-deepgram-flux.py @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07c-interruptible-deepgram-http.py b/examples/foundational/07c-interruptible-deepgram-http.py index c444b5638..f614835e2 100644 --- a/examples/foundational/07c-interruptible-deepgram-http.py +++ b/examples/foundational/07c-interruptible-deepgram-http.py @@ -75,7 +75,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07c-interruptible-deepgram-vad.py b/examples/foundational/07c-interruptible-deepgram-vad.py index fc43bb134..c17880d4a 100644 --- a/examples/foundational/07c-interruptible-deepgram-vad.py +++ b/examples/foundational/07c-interruptible-deepgram-vad.py @@ -68,7 +68,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07c-interruptible-deepgram.py b/examples/foundational/07c-interruptible-deepgram.py index 7dd46124d..17aa4d541 100644 --- a/examples/foundational/07c-interruptible-deepgram.py +++ b/examples/foundational/07c-interruptible-deepgram.py @@ -69,7 +69,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07d-interruptible-elevenlabs-http.py b/examples/foundational/07d-interruptible-elevenlabs-http.py index 8a144dab3..d07e8bb3b 100644 --- a/examples/foundational/07d-interruptible-elevenlabs-http.py +++ b/examples/foundational/07d-interruptible-elevenlabs-http.py @@ -79,7 +79,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index da2a8eb00..2fccccbfa 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07e-interruptible-playht-http.py b/examples/foundational/07e-interruptible-playht-http.py index 0026afc86..e89d5d149 100644 --- a/examples/foundational/07e-interruptible-playht-http.py +++ b/examples/foundational/07e-interruptible-playht-http.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07e-interruptible-playht.py b/examples/foundational/07e-interruptible-playht.py index e854a61cb..2e627d447 100644 --- a/examples/foundational/07e-interruptible-playht.py +++ b/examples/foundational/07e-interruptible-playht.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07f-interruptible-azure-http.py b/examples/foundational/07f-interruptible-azure-http.py index 63971f156..9ae460033 100644 --- a/examples/foundational/07f-interruptible-azure-http.py +++ b/examples/foundational/07f-interruptible-azure-http.py @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07f-interruptible-azure.py b/examples/foundational/07f-interruptible-azure.py index f829063d3..61cff1dcb 100644 --- a/examples/foundational/07f-interruptible-azure.py +++ b/examples/foundational/07f-interruptible-azure.py @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index d3e5ee373..fe3afcc54 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are very knowledgable about dogs. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are very knowledgable about dogs. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index 313843b31..a7a37dc6f 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07i-interruptible-xtts.py b/examples/foundational/07i-interruptible-xtts.py index 029961040..4936fe165 100644 --- a/examples/foundational/07i-interruptible-xtts.py +++ b/examples/foundational/07i-interruptible-xtts.py @@ -75,7 +75,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index 20ddc3f66..788a0101d 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -81,7 +81,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": f"You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": f"You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index daa944cce..f9dfb436b 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -68,7 +68,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07l-interruptible-groq.py b/examples/foundational/07l-interruptible-groq.py index 6f90d0d8d..156e5400e 100644 --- a/examples/foundational/07l-interruptible-groq.py +++ b/examples/foundational/07l-interruptible-groq.py @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07m-interruptible-aws.py b/examples/foundational/07m-interruptible-aws.py index 2d3bb1dac..89dd232fe 100644 --- a/examples/foundational/07m-interruptible-aws.py +++ b/examples/foundational/07m-interruptible-aws.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07n-interruptible-gemini-image.py b/examples/foundational/07n-interruptible-gemini-image.py index 61b8e650a..ff79064c2 100644 --- a/examples/foundational/07n-interruptible-gemini-image.py +++ b/examples/foundational/07n-interruptible-gemini-image.py @@ -94,7 +94,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07n-interruptible-google-http.py b/examples/foundational/07n-interruptible-google-http.py index 136f09d7f..1fa81354b 100644 --- a/examples/foundational/07n-interruptible-google-http.py +++ b/examples/foundational/07n-interruptible-google-http.py @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07n-interruptible-google.py b/examples/foundational/07n-interruptible-google.py index 879ed0d1a..007f3db15 100644 --- a/examples/foundational/07n-interruptible-google.py +++ b/examples/foundational/07n-interruptible-google.py @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index 1b7a1daea..a3d9d5b56 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07p-interruptible-krisp-viva.py b/examples/foundational/07p-interruptible-krisp-viva.py index c7ca15b40..ac85cce46 100644 --- a/examples/foundational/07p-interruptible-krisp-viva.py +++ b/examples/foundational/07p-interruptible-krisp-viva.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07p-interruptible-krisp.py b/examples/foundational/07p-interruptible-krisp.py index 3cbe7f28a..ab10fa9c3 100644 --- a/examples/foundational/07p-interruptible-krisp.py +++ b/examples/foundational/07p-interruptible-krisp.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07q-interruptible-rime-http.py b/examples/foundational/07q-interruptible-rime-http.py index 2e1732c6b..baecdfd3c 100644 --- a/examples/foundational/07q-interruptible-rime-http.py +++ b/examples/foundational/07q-interruptible-rime-http.py @@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07q-interruptible-rime.py b/examples/foundational/07q-interruptible-rime.py index 9ba070151..1cbc84246 100644 --- a/examples/foundational/07q-interruptible-rime.py +++ b/examples/foundational/07q-interruptible-rime.py @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py index 4f451fd53..629f98f58 100644 --- a/examples/foundational/07r-interruptible-riva-nim.py +++ b/examples/foundational/07r-interruptible-riva-nim.py @@ -68,7 +68,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 40759f262..d87c6ebc5 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -53,7 +53,7 @@ You are a helpful LLM in a WebRTC call. Your goals are to be helpful and brief i You are expert at transcribing audio to text. You will receive a mixture of audio and text input. When asked to transcribe what the user said, output an exact, word-for-word transcription. -Your output will be converted to audio so don't include special characters in your answers. +Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Each time you answer, you should respond in three parts. diff --git a/examples/foundational/07t-interruptible-fish.py b/examples/foundational/07t-interruptible-fish.py index 52c4ebbb1..7e756da15 100644 --- a/examples/foundational/07t-interruptible-fish.py +++ b/examples/foundational/07t-interruptible-fish.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07v-interruptible-neuphonic-http.py b/examples/foundational/07v-interruptible-neuphonic-http.py index e22ef734e..e2b26e90d 100644 --- a/examples/foundational/07v-interruptible-neuphonic-http.py +++ b/examples/foundational/07v-interruptible-neuphonic-http.py @@ -76,7 +76,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07v-interruptible-neuphonic.py b/examples/foundational/07v-interruptible-neuphonic.py index c9bb13a70..7cadececb 100644 --- a/examples/foundational/07v-interruptible-neuphonic.py +++ b/examples/foundational/07v-interruptible-neuphonic.py @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07w-interruptible-fal.py b/examples/foundational/07w-interruptible-fal.py index dc572f652..1c4e3a491 100644 --- a/examples/foundational/07w-interruptible-fal.py +++ b/examples/foundational/07w-interruptible-fal.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07x-interruptible-local.py b/examples/foundational/07x-interruptible-local.py index 4778d262f..9b005712f 100644 --- a/examples/foundational/07x-interruptible-local.py +++ b/examples/foundational/07x-interruptible-local.py @@ -56,7 +56,7 @@ async def main(): messages = [ { "role": "system", - "content": "You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07y-interruptible-minimax.py b/examples/foundational/07y-interruptible-minimax.py index 7995ed832..8e7e38723 100644 --- a/examples/foundational/07y-interruptible-minimax.py +++ b/examples/foundational/07y-interruptible-minimax.py @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07z-interruptible-sarvam-http.py b/examples/foundational/07z-interruptible-sarvam-http.py index 4d06affe5..634f0a779 100644 --- a/examples/foundational/07z-interruptible-sarvam-http.py +++ b/examples/foundational/07z-interruptible-sarvam-http.py @@ -80,7 +80,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07z-interruptible-sarvam.py b/examples/foundational/07z-interruptible-sarvam.py index 3123df31d..9590514fe 100644 --- a/examples/foundational/07z-interruptible-sarvam.py +++ b/examples/foundational/07z-interruptible-sarvam.py @@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/08-custom-frame-processor.py b/examples/foundational/08-custom-frame-processor.py index 20da4f876..6c25af0c3 100644 --- a/examples/foundational/08-custom-frame-processor.py +++ b/examples/foundational/08-custom-frame-processor.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/12-describe-image-openai.py b/examples/foundational/12-describe-image-openai.py index 97cb82054..3322c3572 100644 --- a/examples/foundational/12-describe-image-openai.py +++ b/examples/foundational/12-describe-image-openai.py @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are also able to describe images.", }, ] diff --git a/examples/foundational/12a-describe-image-anthropic.py b/examples/foundational/12a-describe-image-anthropic.py index 1690a06bf..e2e1ccbad 100644 --- a/examples/foundational/12a-describe-image-anthropic.py +++ b/examples/foundational/12a-describe-image-anthropic.py @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are also able to describe images.", }, ] diff --git a/examples/foundational/12b-describe-image-aws.py b/examples/foundational/12b-describe-image-aws.py index 1827c8906..6c7f9c3fa 100644 --- a/examples/foundational/12b-describe-image-aws.py +++ b/examples/foundational/12b-describe-image-aws.py @@ -73,7 +73,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are also able to describe images.", }, ] diff --git a/examples/foundational/12c-describe-image-gemini-flash.py b/examples/foundational/12c-describe-image-gemini-flash.py index 9a36785e8..8a388efe6 100644 --- a/examples/foundational/12c-describe-image-gemini-flash.py +++ b/examples/foundational/12c-describe-image-gemini-flash.py @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are also able to describe images.", }, ] diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 42c728bae..dbab11afc 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -120,7 +120,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index 9e626ef14..89fe81ac8 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -106,7 +106,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14d-function-calling-anthropic-video.py b/examples/foundational/14d-function-calling-anthropic-video.py index c933779bb..f3b4ba36d 100644 --- a/examples/foundational/14d-function-calling-anthropic-video.py +++ b/examples/foundational/14d-function-calling-anthropic-video.py @@ -119,7 +119,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", }, ] diff --git a/examples/foundational/14d-function-calling-aws-video.py b/examples/foundational/14d-function-calling-aws-video.py index 392aefca7..0a9a1824f 100644 --- a/examples/foundational/14d-function-calling-aws-video.py +++ b/examples/foundational/14d-function-calling-aws-video.py @@ -126,7 +126,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", }, ] diff --git a/examples/foundational/14d-function-calling-gemini-flash-video.py b/examples/foundational/14d-function-calling-gemini-flash-video.py index c11a4de2e..d166777c9 100644 --- a/examples/foundational/14d-function-calling-gemini-flash-video.py +++ b/examples/foundational/14d-function-calling-gemini-flash-video.py @@ -119,7 +119,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", }, ] diff --git a/examples/foundational/14d-function-calling-moondream-video.py b/examples/foundational/14d-function-calling-moondream-video.py index 83d6ffd66..51f567f4a 100644 --- a/examples/foundational/14d-function-calling-moondream-video.py +++ b/examples/foundational/14d-function-calling-moondream-video.py @@ -120,7 +120,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", }, ] diff --git a/examples/foundational/14d-function-calling-openai-video.py b/examples/foundational/14d-function-calling-openai-video.py index d68ba7f7c..272ea47b7 100644 --- a/examples/foundational/14d-function-calling-openai-video.py +++ b/examples/foundational/14d-function-calling-openai-video.py @@ -119,7 +119,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", }, ] diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 568cf43be..13f121ab4 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -104,7 +104,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index a9bba2c4d..9a75a513e 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -99,7 +99,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index ab72e2bcc..36f2268e4 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -107,7 +107,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 96dce71e6..ee4732d43 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. Start by saying hello.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. Start by saying hello.", }, ] diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 6394bda5d..d02e89e85 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -112,7 +112,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): {"role": "system", "content": "/no_think"}, { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index 93221e1da..a1ecc91ff 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -107,7 +107,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index 32ce47150..01e91e562 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "user", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but try to be brief.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way, but try to be brief.", }, ] diff --git a/examples/foundational/14q-function-calling-qwen.py b/examples/foundational/14q-function-calling-qwen.py index 650142592..69e312cfc 100644 --- a/examples/foundational/14q-function-calling-qwen.py +++ b/examples/foundational/14q-function-calling-qwen.py @@ -105,7 +105,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14r-function-calling-aws.py b/examples/foundational/14r-function-calling-aws.py index 15f7e37a0..bc4b9ffa9 100644 --- a/examples/foundational/14r-function-calling-aws.py +++ b/examples/foundational/14r-function-calling-aws.py @@ -120,7 +120,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14s-function-calling-sambanova.py b/examples/foundational/14s-function-calling-sambanova.py index e425ccd29..6d16daade 100644 --- a/examples/foundational/14s-function-calling-sambanova.py +++ b/examples/foundational/14s-function-calling-sambanova.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14t-function-calling-direct.py b/examples/foundational/14t-function-calling-direct.py index 872cb2a8e..eedf74fa7 100644 --- a/examples/foundational/14t-function-calling-direct.py +++ b/examples/foundational/14t-function-calling-direct.py @@ -106,7 +106,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14u-function-calling-ollama.py b/examples/foundational/14u-function-calling-ollama.py index 3e87c601a..250da11c2 100644 --- a/examples/foundational/14u-function-calling-ollama.py +++ b/examples/foundational/14u-function-calling-ollama.py @@ -122,7 +122,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14v-function-calling-openai.py b/examples/foundational/14v-function-calling-openai.py index d259e7131..cf3039dde 100644 --- a/examples/foundational/14v-function-calling-openai.py +++ b/examples/foundational/14v-function-calling-openai.py @@ -128,7 +128,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14w-function-calling-mistral.py b/examples/foundational/14w-function-calling-mistral.py index ce61085e2..e1fd1b2eb 100644 --- a/examples/foundational/14w-function-calling-mistral.py +++ b/examples/foundational/14w-function-calling-mistral.py @@ -116,7 +116,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14x-function-calling-openpipe.py b/examples/foundational/14x-function-calling-openpipe.py index 3f2537bb7..78ff796c9 100644 --- a/examples/foundational/14x-function-calling-openpipe.py +++ b/examples/foundational/14x-function-calling-openpipe.py @@ -126,7 +126,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/16-gpu-container-local-bot.py b/examples/foundational/16-gpu-container-local-bot.py index 90ff271c6..39ad7d345 100644 --- a/examples/foundational/16-gpu-container-local-bot.py +++ b/examples/foundational/16-gpu-container-local-bot.py @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index cc36d891d..6d0a94460 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 93c1fa438..4ecda106d 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -98,7 +98,7 @@ async def load_conversation(params: FunctionCallParams): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index e8822bbc6..77dc3627c 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -100,7 +100,7 @@ async def load_conversation(params: FunctionCallParams): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a succinct, creative and helpful way. Prefer responses that are one sentence long unless you are asked for a longer or more detailed response.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a succinct, creative and helpful way. Prefer responses that are one sentence long unless you are asked for a longer or more detailed response.", }, {"role": "user", "content": "Start the call by saying the word 'hello'. Say only that word."}, # {"role": "user", "content": ""}, diff --git a/examples/foundational/21-tavus-transport.py b/examples/foundational/21-tavus-transport.py index 4aabaa23e..0e68c8507 100644 --- a/examples/foundational/21-tavus-transport.py +++ b/examples/foundational/21-tavus-transport.py @@ -61,7 +61,7 @@ async def main(): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/21a-tavus-video-service.py b/examples/foundational/21a-tavus-video-service.py index 0aa2f7e17..d2421eae4 100644 --- a/examples/foundational/21a-tavus-video-service.py +++ b/examples/foundational/21a-tavus-video-service.py @@ -80,7 +80,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 98ad23c0b..2ccadbd2b 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -142,7 +142,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 9b3ff1d7a..7e0c85284 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -327,7 +327,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 461fab08d..db0740a37 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -258,7 +258,7 @@ Output: YES Output: NO """ -conversational_system_message = """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. +conversational_system_message = """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. Please be very concise in your responses. Unless you are explicitly asked to do otherwise, give me the shortest complete answer possible without unnecessary elaboration. Generally you should answer with a single sentence. """ diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 5819c9cb9..60ea7d381 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -340,7 +340,7 @@ Output: NO conversation_system_instruction = """You are a helpful assistant participating in a voice converation. -Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. +Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. If you know that a number string is a phone number from the context of the conversation, write it as a phone number. For example 210-333-4567. diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index b34947c2b..e769c9b2a 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -90,7 +90,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index 15eae179c..0e1413cd1 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -45,7 +45,7 @@ load_dotenv(override=True) # conversation_system_message = """ You are a helpful LLM in a WebRTC call. Your goals are to be helpful and brief in your responses. Respond with one or two sentences at most, unless you are asked to -respond at more length. Your output will be converted to audio so don't include special characters in your answers. +respond at more length. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. """ # diff --git a/examples/foundational/26-gemini-live.py b/examples/foundational/26-gemini-live.py index dd5dfcffc..924c3628b 100644 --- a/examples/foundational/26-gemini-live.py +++ b/examples/foundational/26-gemini-live.py @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): system_instruction = f""" You are a helpful AI assistant. Your goal is to demonstrate your capabilities in a helpful and engaging way. - Your output will be converted to audio so don't include special characters in your answers. + Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. """ diff --git a/examples/foundational/26d-gemini-live-text.py b/examples/foundational/26d-gemini-live-text.py index fc9f68bcb..111f5a6cf 100644 --- a/examples/foundational/26d-gemini-live-text.py +++ b/examples/foundational/26d-gemini-live-text.py @@ -38,7 +38,7 @@ SYSTEM_INSTRUCTION = f""" Your goal is to demonstrate your capabilities in a succinct way. -Your output will be converted to audio so don't include special characters in your answers. +Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. Keep your responses brief. One or two sentences at most. """ diff --git a/examples/foundational/26f-gemini-live-files-api.py b/examples/foundational/26f-gemini-live-files-api.py index e8b38ba6d..855cd151c 100644 --- a/examples/foundational/26f-gemini-live-files-api.py +++ b/examples/foundational/26f-gemini-live-files-api.py @@ -105,7 +105,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - Answer questions about what's in the document - Use the information from the document in our conversation - Your output will be converted to audio so don't include special characters in your answers. + Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Be friendly and demonstrate your ability to work with the uploaded file. """ diff --git a/examples/foundational/26g-gemini-live-groundingMetadata.py b/examples/foundational/26g-gemini-live-groundingMetadata.py index 553539ab2..468fcd678 100644 --- a/examples/foundational/26g-gemini-live-groundingMetadata.py +++ b/examples/foundational/26g-gemini-live-groundingMetadata.py @@ -63,7 +63,7 @@ You should use Google Search for: Always be proactive about using search when the user asks about anything that could benefit from real-time information. -Your output will be converted to audio so don't include special characters in your answers. +Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way, always using search for current information. """ diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 348cf117b..c2c3f35ce 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/28-transcription-processor.py b/examples/foundational/28-transcription-processor.py index 6aa3168ac..21d800198 100644 --- a/examples/foundational/28-transcription-processor.py +++ b/examples/foundational/28-transcription-processor.py @@ -134,7 +134,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way. Say hello.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative, helpful, and brief way. Say hello.", }, ] diff --git a/examples/foundational/29-turn-tracking-observer.py b/examples/foundational/29-turn-tracking-observer.py index 83faa71dc..6656ed83a 100644 --- a/examples/foundational/29-turn-tracking-observer.py +++ b/examples/foundational/29-turn-tracking-observer.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index 54318884e..6e02aee37 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -119,7 +119,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/36-user-email-gathering.py b/examples/foundational/36-user-email-gathering.py index 4cf20b750..e69f6bee4 100644 --- a/examples/foundational/36-user-email-gathering.py +++ b/examples/foundational/36-user-email-gathering.py @@ -109,9 +109,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): { "role": "system", # Cartesia - "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers. If the user provides one or more email addresses confirm them with the user. Enclose all emails with tags, for example a@a.com.", + "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. If the user provides one or more email addresses confirm them with the user. Enclose all emails with tags, for example a@a.com.", # Rime spell() - # "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers. If the user provides one or more email addresses confirm them with the user. Enclose all emails with spell(), for example spell(a@a.com).", + # "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. If the user provides one or more email addresses confirm them with the user. Enclose all emails with spell(), for example spell(a@a.com).", }, ] diff --git a/examples/foundational/38-smart-turn-fal.py b/examples/foundational/38-smart-turn-fal.py index 03792c11d..7d01f41f9 100644 --- a/examples/foundational/38-smart-turn-fal.py +++ b/examples/foundational/38-smart-turn-fal.py @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/38a-smart-turn-local-coreml.py b/examples/foundational/38a-smart-turn-local-coreml.py index fc11b4137..ca13be6f2 100644 --- a/examples/foundational/38a-smart-turn-local-coreml.py +++ b/examples/foundational/38a-smart-turn-local-coreml.py @@ -94,7 +94,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/38b-smart-turn-local.py b/examples/foundational/38b-smart-turn-local.py index 8e8136243..d00e0ecb3 100644 --- a/examples/foundational/38b-smart-turn-local.py +++ b/examples/foundational/38b-smart-turn-local.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/39-mcp-stdio.py b/examples/foundational/39-mcp-stdio.py index f8df7307f..f707866b9 100644 --- a/examples/foundational/39-mcp-stdio.py +++ b/examples/foundational/39-mcp-stdio.py @@ -158,7 +158,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. - Your output will be converted to audio so don't include special characters in your answers. + Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. Don't overexplain what you are doing. Just respond with short sentences when you are carrying out tool calls. diff --git a/examples/foundational/39a-mcp-run-sse.py b/examples/foundational/39a-mcp-run-sse.py index 366ebfae2..90de1b91e 100644 --- a/examples/foundational/39a-mcp-run-sse.py +++ b/examples/foundational/39a-mcp-run-sse.py @@ -90,7 +90,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. You have access to a number of tools provided by mcp.run. Use any and all tools to help users. - Your output will be converted to audio so don't include special characters in your answers. + Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. When asked for today's date, use 'https://www.datetoday.net/'. Don't overexplain what you are doing. diff --git a/examples/foundational/39b-multiple-mcp.py b/examples/foundational/39b-multiple-mcp.py index e56744203..21e5bff60 100644 --- a/examples/foundational/39b-multiple-mcp.py +++ b/examples/foundational/39b-multiple-mcp.py @@ -136,7 +136,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. - Your output will be converted to audio so don't include special characters in your answers. + Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. Don't overexplain what you are doing. Just respond with short sentences when you are carrying out tool calls. diff --git a/examples/foundational/39c-mcp-run-http.py b/examples/foundational/39c-mcp-run-http.py index 9d64bb81b..8e704aaa6 100644 --- a/examples/foundational/39c-mcp-run-http.py +++ b/examples/foundational/39c-mcp-run-http.py @@ -96,7 +96,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): You are a helpful LLM in a WebRTC call. Your goal is to answer questions about the user's GitHub repositories and account. You have access to a number of tools provided by Github. Use any and all tools to help users. - Your output will be converted to audio so don't include special characters in your answers. + Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Don't overexplain what you are doing. Just respond with short sentences when you are carrying out tool calls. """ diff --git a/examples/foundational/39d-mcp-run-http-gemini-live.py b/examples/foundational/39d-mcp-run-http-gemini-live.py index b4dfbb01e..ff2c7649a 100644 --- a/examples/foundational/39d-mcp-run-http-gemini-live.py +++ b/examples/foundational/39d-mcp-run-http-gemini-live.py @@ -94,7 +94,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): You are a helpful LLM in a WebRTC call. Your goal is to answer questions about the user's GitHub repositories and account. You have access to a number of tools provided by Github. Use any and all tools to help users. - Your output will be converted to audio so don't include special characters in your answers. + Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Don't overexplain what you are doing. Just respond with short sentences when you are carrying out tool calls. """ diff --git a/examples/foundational/42-interruption-config.py b/examples/foundational/42-interruption-config.py index c3c899b89..2306ad936 100644 --- a/examples/foundational/42-interruption-config.py +++ b/examples/foundational/42-interruption-config.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/43-heygen-transport.py b/examples/foundational/43-heygen-transport.py index 78aab9622..efccbd808 100644 --- a/examples/foundational/43-heygen-transport.py +++ b/examples/foundational/43-heygen-transport.py @@ -60,7 +60,7 @@ async def main(): messages = [ { "role": "system", - "content": "You are a helpful assistant. Your output will be converted to audio so don't include special characters in your answers. Be succinct and respond to what the user said in a creative and helpful way.", + "content": "You are a helpful assistant. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Be succinct and respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/43a-heygen-video-service.py b/examples/foundational/43a-heygen-video-service.py index 9f25a0cfb..5f9ddce21 100644 --- a/examples/foundational/43a-heygen-video-service.py +++ b/examples/foundational/43a-heygen-video-service.py @@ -83,7 +83,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful assistant. Your output will be converted to audio so don't include special characters in your answers. Be succinct and respond to what the user said in a creative and helpful way.", + "content": "You are a helpful assistant. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Be succinct and respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index 257441a40..fb612f877 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/45-before-and-after-events.py b/examples/foundational/45-before-and-after-events.py index fd7bdfa5b..b6ad5d807 100644 --- a/examples/foundational/45-before-and-after-events.py +++ b/examples/foundational/45-before-and-after-events.py @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/46-video-processing.py b/examples/foundational/46-video-processing.py index 159da270a..c6311db3b 100644 --- a/examples/foundational/46-video-processing.py +++ b/examples/foundational/46-video-processing.py @@ -89,7 +89,7 @@ SYSTEM_INSTRUCTION = f""" Your goal is to demonstrate your capabilities in a succinct way. -Your output will be converted to audio so don't include special characters in your answers. +Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. Keep your responses brief. One or two sentences at most. """ diff --git a/examples/foundational/47-sentry-metrics.py b/examples/foundational/47-sentry-metrics.py index ae7b7a59d..958557f81 100644 --- a/examples/foundational/47-sentry-metrics.py +++ b/examples/foundational/47-sentry-metrics.py @@ -85,7 +85,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/48-service-switcher.py b/examples/foundational/48-service-switcher.py index 8e0f8db85..c13ff7273 100644 --- a/examples/foundational/48-service-switcher.py +++ b/examples/foundational/48-service-switcher.py @@ -129,7 +129,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", }, ] tools = ToolsSchema(standard_tools=[weather_function, get_restaurant_recommendation]) From 498e9ca4f62e83b143244bdf6af1edb8c64ab942 Mon Sep 17 00:00:00 2001 From: gokuljs Date: Thu, 13 Nov 2025 04:33:22 +0530 Subject: [PATCH 025/110] Add support for Hindi language in RIme TTS service --- src/pipecat/services/rime/tts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 0ac37c471..b51f9e3ec 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -60,6 +60,7 @@ def language_to_rime_language(language: Language) -> str: Language.FR: "fra", Language.EN: "eng", Language.ES: "spa", + Language.HI: "hin", } return resolve_language(language, LANGUAGE_MAP, use_base_code=False) From fe25465987aee96dd2283a74edcf21bc22a31af4 Mon Sep 17 00:00:00 2001 From: gokuljs Date: Thu, 13 Nov 2025 07:16:36 +0530 Subject: [PATCH 026/110] changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a43553f0c..f8f8bed9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 services that subclass `TTSService` can indicate whether the text in the `TTSTextFrame`s they push already contain any necessary inter-frame spaces. +### Changed + +- Added Hindi support for Rime TTS services. + ### Fixed - Fixed subtle issue of assistant context messages ending up with double spaces From a7b2052b38f054af4c30e6fb2c00f5b2146f71bc Mon Sep 17 00:00:00 2001 From: Corvin Jaedicke Date: Thu, 13 Nov 2025 14:20:35 +0100 Subject: [PATCH 027/110] add ai-coustics VAD --- .../07ad-interruptible-aicoustics.py | 51 +++--- pyproject.toml | 4 + src/pipecat/audio/filters/aic_filter.py | 52 ++++++ src/pipecat/audio/vad/aic_vad.py | 158 ++++++++++++++++++ uv.lock | 36 ++-- 5 files changed, 261 insertions(+), 40 deletions(-) create mode 100644 src/pipecat/audio/vad/aic_vad.py diff --git a/examples/foundational/07ad-interruptible-aicoustics.py b/examples/foundational/07ad-interruptible-aicoustics.py index aa647ff33..0f6832a28 100644 --- a/examples/foundational/07ad-interruptible-aicoustics.py +++ b/examples/foundational/07ad-interruptible-aicoustics.py @@ -15,7 +15,6 @@ from loguru import logger from pipecat.audio.filters.aic_filter import AICFilter from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline @@ -48,7 +47,7 @@ def _create_aic_filter() -> AICFilter: return AICFilter( license_key=license_key, - enhancement_level=1.0, + enhancement_level=0.5, ) @@ -56,27 +55,33 @@ def _create_aic_filter() -> AICFilter: # instantiated. The function will be called when the desired transport gets # selected. transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), + "daily": lambda: ( + lambda aic: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), + "twilio": lambda: ( + lambda aic: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), + "webrtc": lambda: ( + lambda aic: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), } diff --git a/pyproject.toml b/pyproject.toml index 7b382f112..4e7a6130d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -128,6 +128,7 @@ dev = [ "setuptools~=78.1.1", "setuptools_scm~=8.3.1", "python-dotenv>=1.0.1,<2.0.0", + "pipecat-ai[aic,daily,deepgram,local-smart-turn-v3,openai,runner,silero,webrtc]", ] docs = [ @@ -205,3 +206,6 @@ convention = "google" command_line = "--module pytest" source = [ "src" ] omit = [ "*/tests/*" ] + +[tool.uv.sources] +pipecat-ai = { workspace = true } diff --git a/src/pipecat/audio/filters/aic_filter.py b/src/pipecat/audio/filters/aic_filter.py index 3f1af2f57..2f4699912 100644 --- a/src/pipecat/audio/filters/aic_filter.py +++ b/src/pipecat/audio/filters/aic_filter.py @@ -68,6 +68,58 @@ class AICFilter(BaseAudioFilter): # Model will be created in start() since the API now requires sample_rate self._aic = None + def get_vad_factory(self): + """Return a zero-arg factory that will create the VAD once the model exists. + + Returns: + A zero-argument callable that, when invoked, returns an initialized + VoiceActivityDetector bound to the underlying AIC model. Raises a + RuntimeError if the model has not been initialized (i.e. start() + has not been called successfully). + """ + + def _factory(): + if self._aic is None: + raise RuntimeError("AIC model not initialized yet. Call start(sample_rate) first.") + return self._aic.create_vad() + + return _factory + + def create_vad_analyzer( + self, + *, + lookback_buffer_size: Optional[float] = None, + sensitivity: Optional[float] = None, + ): + """Return an analyzer that will lazily instantiate the AIC VAD when ready. + + AIC VAD parameters: + - lookback_buffer_size: + Number of window-length audio buffers used as a lookback buffer. + Higher values increase prediction stability but add latency. + Range: 1.0 .. 20.0, Default (SDK): 6.0 + - sensitivity: + Energy threshold sensitivity. Energy threshold = 10 ** (-sensitivity). + Range: 1.0 .. 15.0, Default (SDK): 6.0 + + Args: + lookback_buffer_size: Optional lookback buffer size to configure on the VAD. + Range: 1.0 .. 20.0. If None, SDK default is used. + sensitivity: Optional sensitivity (energy threshold) to configure on the VAD. + Range: 1.0 .. 15.0. If None, SDK default is used. + + Returns: + A lazily-initialized AICVADAnalyzer that will bind to the VAD backend + once the filter's model has been created (after start(sample_rate)). + """ + from pipecat.audio.vad.aic_vad import AICVADAnalyzer + + return AICVADAnalyzer( + vad_factory=self.get_vad_factory(), + lookback_buffer_size=lookback_buffer_size, + sensitivity=sensitivity, + ) + async def start(self, sample_rate: int): """Initialize the filter with the transport's sample rate. diff --git a/src/pipecat/audio/vad/aic_vad.py b/src/pipecat/audio/vad/aic_vad.py new file mode 100644 index 000000000..4907e4f55 --- /dev/null +++ b/src/pipecat/audio/vad/aic_vad.py @@ -0,0 +1,158 @@ +"""AIC-integrated VAD analyzer that lazily binds to the AIC SDK backend. + +This analyzer queries the backend's is_speech_detected() and maps it to a float +confidence (1.0/0.0). It uses 10 ms windows based on the sample rate and applies +optional AIC VAD parameters (lookback_buffer_size, sensitivity) when available. +""" + +from typing import Any, Callable, Optional + +from loguru import logger + +from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams + +try: + from aic import AICVadParameter +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.") + raise Exception(f"Missing module: {e}") + + +class AICVADAnalyzer(VADAnalyzer): + """VAD analyzer that lazily instantiates the AIC VoiceActivityDetector via a factory. + + The analyzer can be constructed before the AIC Model exists. Once the filter has + started and the Model is available, the provided factory will succeed and the + backend VAD will be created. We then switch to single-sample updates where + num_frames_required() returns 1 and confidence is derived from the backend's + boolean is_speech_detected() state. + + AIC VAD runtime parameters: + - lookback_buffer_size: + Controls the lookback buffer size used by the VAD, i.e. the number of + window-length audio buffers used as a lookback buffer. Larger values improve + stability but increase latency. + Range: 1.0 .. 20.0 + Default (SDK): 6.0 + - sensitivity: + Controls the energy threshold sensitivity. Higher values make the detector + less sensitive (require more energy to count as speech). + Range: 1.0 .. 15.0 + Formula: Energy threshold = 10 ** (-sensitivity) + Default (SDK): 6.0 + """ + + def __init__( + self, + *, + vad_factory: Optional[Callable[[], Any]] = None, + lookback_buffer_size: Optional[float] = None, + sensitivity: Optional[float] = None, + ): + """Create an AIC VAD analyzer. + + Args: + vad_factory: + Zero-arg callable that returns an initialized AIC VoiceActivityDetector. + This may raise until the filter's Model has been created; the analyzer + will retry on set_sample_rate/first use. + lookback_buffer_size: + Optional override for AIC VAD lookback buffer size. + Range: 1.0 .. 20.0. Larger values increase stability at the cost of latency. + If None, the SDK default (6.0) is used. + sensitivity: + Optional override for AIC VAD sensitivity (energy threshold). + Range: 1.0 .. 15.0. Energy threshold = 10 ** (-sensitivity). + If None, the SDK default (6.0) is used. + """ + # Use fixed VAD parameters for AIC: no user override + fixed_params = VADParams(confidence=0.5, start_secs=0.0, stop_secs=0.0, min_volume=0.0) + super().__init__(sample_rate=None, params=fixed_params) + self._vad_factory = vad_factory + self._backend_vad: Optional[Any] = None + self._pending_lookback: Optional[float] = lookback_buffer_size + self._pending_sensitivity: Optional[float] = sensitivity + + def bind_vad_factory(self, vad_factory: Callable[[], Any]): + """Attach or replace the factory post-construction.""" + self._vad_factory = vad_factory + self._ensure_backend_initialized() + + def _apply_backend_params(self): + """Apply optional AIC VAD parameters if available.""" + if self._backend_vad is None or AICVadParameter is None: + return + try: + if self._pending_lookback is not None: + self._backend_vad.set_parameter( + AICVadParameter.LOOKBACK_BUFFER_SIZE, float(self._pending_lookback) + ) + if self._pending_sensitivity is not None: + self._backend_vad.set_parameter( + AICVadParameter.SENSITIVITY, float(self._pending_sensitivity) + ) + except Exception as e: # noqa: BLE001 + logger.debug(f"AIC VAD parameter application deferred/failed: {e}") + + def _ensure_backend_initialized(self): + if self._backend_vad is not None: + return + if not self._vad_factory: + return + try: + self._backend_vad = self._vad_factory() + self._apply_backend_params() + # With backend ready, recompute internal frame sizing + super().set_params(self._params) + logger.debug("AIC VAD backend initialized in analyzer.") + except Exception as e: # noqa: BLE001 + # Filter may not be started yet; try again later + logger.debug(f"Deferring AIC VAD backend initialization: {e}") + + def set_sample_rate(self, sample_rate: int): + """Set the sample rate for audio processing. + + Args: + sample_rate: Audio sample rate in Hz. + """ + # Set rate and attempt backend initialization once we know SR + self._sample_rate = self._init_sample_rate or sample_rate + self._ensure_backend_initialized() + # Ensure params are initialized even if backend not ready yet + try: + super().set_params(self._params) + except Exception: + pass + + def num_frames_required(self) -> int: + """Get the number of audio frames required for analysis. + + Returns: + Number of frames needed for VAD processing. + """ + # Use 10 ms windows based on sample rate + return int(self.sample_rate * 0.01) if self.sample_rate > 0 else 160 + + def voice_confidence(self, buffer: bytes) -> float: + """Calculate voice activity confidence for the given audio buffer. + + Args: + buffer: Audio buffer to analyze. + + Returns: + Voice confidence score is 0.0 or 1.0. + """ + # Ensure backend exists (filter might have started since last call) + self._ensure_backend_initialized() + if self._backend_vad is None: + return 0.0 + + # We do not need to analyze 'buffer' here since the model's VAD is updated + # as part of the enhancement pipeline. Simply query the boolean and map it. + try: + is_speech = self._backend_vad.is_speech_detected() + return 1.0 if is_speech else 0.0 + except Exception as e: # noqa: BLE001 + logger.error(f"AIC VAD inference error: {e}") + return 0.0 diff --git a/uv.lock b/uv.lock index eabe03714..7fb531dea 100644 --- a/uv.lock +++ b/uv.lock @@ -4624,6 +4624,7 @@ dev = [ { name = "coverage" }, { name = "grpcio-tools" }, { name = "pip-tools" }, + { name = "pipecat-ai", extra = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"] }, { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, @@ -4697,23 +4698,23 @@ requires-dist = [ { name = "opentelemetry-sdk", marker = "extra == 'tracing'", specifier = ">=1.33.0" }, { name = "ormsgpack", marker = "extra == 'fish'", specifier = "~=1.7.0" }, { name = "pillow", specifier = ">=11.1.0,<12" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'", editable = "." }, { name = "pipecat-ai-krisp", marker = "extra == 'krisp'", specifier = "~=0.4.0" }, { name = "pipecat-ai-small-webrtc-prebuilt", marker = "extra == 'runner'", specifier = ">=1.0.0" }, { name = "protobuf", specifier = "~=5.29.3" }, @@ -4753,6 +4754,7 @@ dev = [ { name = "coverage", specifier = "~=7.9.1" }, { name = "grpcio-tools", specifier = "~=1.67.1" }, { name = "pip-tools", specifier = "~=7.4.1" }, + { name = "pipecat-ai", extras = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"], editable = "." }, { name = "pre-commit", specifier = "~=4.2.0" }, { name = "pyright", specifier = ">=1.1.404,<1.2" }, { name = "pytest", specifier = "~=8.4.1" }, From 2fab3e228638952705f1b30206c10caab965da4b Mon Sep 17 00:00:00 2001 From: Corvin Jaedicke Date: Thu, 13 Nov 2025 14:39:26 +0100 Subject: [PATCH 028/110] fix formatting --- pyproject.toml | 82 ++++++++++++++++++++++++-------------------------- uv.lock | 42 ++++++++++++-------------- 2 files changed, 59 insertions(+), 65 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4e7a6130d..e313c789d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,23 +1,23 @@ [build-system] -requires = [ "setuptools>=64", "setuptools_scm>=8" ] +requires = ["setuptools>=64", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" [project] name = "pipecat-ai" -dynamic = [ "version" ] +dynamic = ["version"] description = "An open source framework for voice (and multimodal) assistants" license = "BSD-2-Clause" -license-files = [ "LICENSE" ] +license-files = ["LICENSE"] readme = "README.md" requires-python = ">=3.10" -keywords = [ "webrtc", "audio", "video", "ai" ] +keywords = ["webrtc", "audio", "video", "ai"] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Topic :: Communications :: Conferencing", "Topic :: Multimedia :: Sound/Audio", "Topic :: Multimedia :: Video", - "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Artificial Intelligence" ] dependencies = [ "aiofiles>=24.1.0,<25", @@ -51,24 +51,24 @@ assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] aws = [ "aioboto3~=15.0.0", "pipecat-ai[websockets-base]" ] aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.1.1; python_version>='3.12'" ] -azure = [ "azure-cognitiveservices-speech~=1.42.0" ] +azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] -cerebras = [ ] -deepseek = [ ] +cerebras = [] +deepseek = [] daily = [ "daily-python~=0.21.0" ] deepgram = [ "deepgram-sdk~=4.7.0" ] elevenlabs = [ "pipecat-ai[websockets-base]" ] fal = [ "fal-client~=0.5.9" ] -fireworks = [ ] +fireworks = [] fish = [ "ormsgpack~=1.7.0", "pipecat-ai[websockets-base]" ] gladia = [ "pipecat-ai[websockets-base]" ] google = [ "google-cloud-speech>=2.33.0,<3", "google-cloud-texttospeech>=2.31.0,<3", "google-genai>=1.41.0,<2", "pipecat-ai[websockets-base]" ] -grok = [ ] +grok = [] groq = [ "groq~=0.23.0" ] gstreamer = [ "pygobject~=3.50.0" ] heygen = [ "livekit>=1.0.13", "pipecat-ai[websockets-base]" ] hume = [ "hume>=0.11.2" ] -inworld = [ ] +inworld = [] krisp = [ "pipecat-ai-krisp~=0.4.0" ] koala = [ "pvkoala~=2.0.3" ] langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-openai~=0.3.9" ] @@ -77,35 +77,35 @@ lmnt = [ "pipecat-ai[websockets-base]" ] local = [ "pyaudio~=0.2.14" ] mcp = [ "mcp[cli]>=1.11.0,<2" ] mem0 = [ "mem0ai~=0.1.94" ] -mistral = [ ] +mistral = [] mlx-whisper = [ "mlx-whisper~=0.4.2" ] moondream = [ "accelerate~=1.10.0", "einops~=0.8.0", "pyvips[binary]~=3.0.0", "timm~=1.0.13", "transformers>=4.48.0" ] -nim = [ ] +nim = [] neuphonic = [ "pipecat-ai[websockets-base]" ] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "pipecat-ai[websockets-base]" ] openpipe = [ "openpipe>=4.50.0,<6" ] -openrouter = [ ] -perplexity = [ ] +openrouter = [] +perplexity = [] playht = [ "pipecat-ai[websockets-base]" ] -qwen = [ ] +qwen = [] rime = [ "pipecat-ai[websockets-base]" ] riva = [ "nvidia-riva-client~=2.21.1" ] -runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.122.0", "pipecat-ai-small-webrtc-prebuilt>=1.0.0" ] -sambanova = [ ] +runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.122.0", "pipecat-ai-small-webrtc-prebuilt>=1.0.0"] +sambanova = [] sarvam = [ "sarvamai==0.1.21", "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk>=2.28.0,<3" ] local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "torchaudio>=2.5.0,<3" ] local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] -remote-smart-turn = [ ] +remote-smart-turn = [] silero = [ "onnxruntime>=1.20.1,<2" ] -simli = [ "simli-ai~=0.1.25" ] +simli = [ "simli-ai~=0.1.25"] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.1" ] speechmatics = [ "speechmatics-rt>=0.5.0" ] strands = [ "strands-agents>=1.9.1,<2" ] -tavus = [ ] -together = [ ] +tavus=[] +together = [] tracing = [ "opentelemetry-sdk>=1.33.0", "opentelemetry-api>=1.33.0", "opentelemetry-instrumentation>=0.54b0" ] ultravox = [ "transformers>=4.48.0", "vllm>=0.9.0" ] webrtc = [ "aiortc>=1.13.0,<2", "opencv-python>=4.11.0.86,<5" ] @@ -128,7 +128,6 @@ dev = [ "setuptools~=78.1.1", "setuptools_scm~=8.3.1", "python-dotenv>=1.0.1,<2.0.0", - "pipecat-ai[aic,daily,deepgram,local-smart-turn-v3,openai,runner,silero,webrtc]", ] docs = [ @@ -140,10 +139,10 @@ docs = [ ] [tool.setuptools.packages.find] -where = [ "src" ] +where = ["src"] [tool.setuptools.package-data] -"pipecat" = [ "py.typed" ] +"pipecat" = ["py.typed"] "pipecat.audio.dtmf" = [ "src/pipecat/audio/dtmf/dtmf-0.wav", "src/pipecat/audio/dtmf/dtmf-1.wav", @@ -158,13 +157,13 @@ where = [ "src" ] "src/pipecat/audio/dtmf/dtmf-pound.wav", "src/pipecat/audio/dtmf/dtmf-star.wav", ] -"pipecat.services.aws_nova_sonic" = [ "src/pipecat/services/aws_nova_sonic/ready.wav" ] -"pipecat.audio.turn.smart_turn.data" = [ "src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx" ] +"pipecat.services.aws_nova_sonic" = ["src/pipecat/services/aws_nova_sonic/ready.wav"] +"pipecat.audio.turn.smart_turn.data" = ["src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx"] [tool.pytest.ini_options] addopts = "--verbose" -testpaths = [ "tests" ] -pythonpath = [ "src" ] +testpaths = ["tests"] +pythonpath = ["src"] asyncio_default_fixture_loop_scope = "function" filterwarnings = [ "ignore:'audioop' is deprecated:DeprecationWarning", @@ -175,7 +174,7 @@ local_scheme = "no-local-version" fallback_version = "0.0.0-dev" [tool.ruff] -exclude = [ ".git", "*_pb2.py" ] +exclude = [".git", "*_pb2.py"] line-length = 100 [tool.ruff.lint] @@ -184,28 +183,25 @@ select = [ "I", # Import rules ] ignore = [ - "D105", # Missing docstring in magic methods (__str__, __repr__, etc.) + "D105", # Missing docstring in magic methods (__str__, __repr__, etc.) ] [tool.ruff.lint.per-file-ignores] # Skip docstring checks for non-source code -"examples/**/*.py" = [ "D" ] -"tests/**/*.py" = [ "D" ] -"scripts/**/*.py" = [ "D" ] -"docs/**/*.py" = [ "D" ] +"examples/**/*.py" = ["D"] +"tests/**/*.py" = ["D"] +"scripts/**/*.py" = ["D"] +"docs/**/*.py" = ["D"] # Skip D104 (missing docstring in public package) for __init__.py files -"**/__init__.py" = [ "D104" ] +"**/__init__.py" = ["D104"] # Skip specific rules for generated protobuf files -"**/*_pb2.py" = [ "D" ] -"src/pipecat/services/__init__.py" = [ "D" ] +"**/*_pb2.py" = ["D"] +"src/pipecat/services/__init__.py" = ["D"] [tool.ruff.lint.pydocstyle] convention = "google" [tool.coverage.run] command_line = "--module pytest" -source = [ "src" ] -omit = [ "*/tests/*" ] - -[tool.uv.sources] -pipecat-ai = { workspace = true } +source = ["src"] +omit = ["*/tests/*"] diff --git a/uv.lock b/uv.lock index 7fb531dea..0dc6c74f8 100644 --- a/uv.lock +++ b/uv.lock @@ -36,12 +36,12 @@ wheels = [ [[package]] name = "aic-sdk" -version = "1.1.0" +version = "1.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/99/83/bf38b95d98c67b8ebc574fb4a4f23c07a3740b51992d7522976173d30b98/aic_sdk-1.1.0.tar.gz", hash = "sha256:04e08df695581c8cb4db8acca20e73815e9f449e7bd08e0162fd55518c727963", size = 34954, upload-time = "2025-11-11T20:45:24.25Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/90/b02e853e863c303f8456c689b42ac24ad403b781adc9642d0a91ed4bed7e/aic_sdk-1.0.2.tar.gz", hash = "sha256:239097dd3aaa8a8a0fd7542b75d2510cb34144caec796370639b7c636acbc56e", size = 32059, upload-time = "2025-08-24T09:20:03.9Z" } [[package]] name = "aioboto3" @@ -4624,7 +4624,6 @@ dev = [ { name = "coverage" }, { name = "grpcio-tools" }, { name = "pip-tools" }, - { name = "pipecat-ai", extra = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"] }, { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, @@ -4648,7 +4647,7 @@ docs = [ [package.metadata] requires-dist = [ { name = "accelerate", marker = "extra == 'moondream'", specifier = "~=1.10.0" }, - { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.1.0" }, + { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.0.1" }, { name = "aioboto3", marker = "extra == 'aws'", specifier = "~=15.0.0" }, { name = "aiofiles", specifier = ">=24.1.0,<25" }, { name = "aiohttp", specifier = ">=3.11.12,<4" }, @@ -4698,23 +4697,23 @@ requires-dist = [ { name = "opentelemetry-sdk", marker = "extra == 'tracing'", specifier = ">=1.33.0" }, { name = "ormsgpack", marker = "extra == 'fish'", specifier = "~=1.7.0" }, { name = "pillow", specifier = ">=11.1.0,<12" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'" }, { name = "pipecat-ai-krisp", marker = "extra == 'krisp'", specifier = "~=0.4.0" }, { name = "pipecat-ai-small-webrtc-prebuilt", marker = "extra == 'runner'", specifier = ">=1.0.0" }, { name = "protobuf", specifier = "~=5.29.3" }, @@ -4754,7 +4753,6 @@ dev = [ { name = "coverage", specifier = "~=7.9.1" }, { name = "grpcio-tools", specifier = "~=1.67.1" }, { name = "pip-tools", specifier = "~=7.4.1" }, - { name = "pipecat-ai", extras = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"], editable = "." }, { name = "pre-commit", specifier = "~=4.2.0" }, { name = "pyright", specifier = ">=1.1.404,<1.2" }, { name = "pytest", specifier = "~=8.4.1" }, From 0e37658f8d59e989b45312ebc16e80bd246a01db Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 12 Nov 2025 11:06:45 -0500 Subject: [PATCH 029/110] Add ElevenLabsRealtimeSTTService --- CHANGELOG.md | 3 + .../07d-interruptible-elevenlabs.py | 4 +- src/pipecat/services/elevenlabs/stt.py | 480 +++++++++++++++++- 3 files changed, 483 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8f8bed9b..de9cf3328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT + service from ElevenLabs. + - Added a `TTSService.includes_inter_frame_spaces` property getter, so that TTS services that subclass `TTSService` can indicate whether the text in the `TTSTextFrame`s they push already contain any necessary inter-frame spaces. diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index da2a8eb00..2d14a4d77 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -22,7 +22,7 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.elevenlabs.stt import ElevenLabsRealtimeSTTService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams @@ -60,7 +60,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = ElevenLabsRealtimeSTTService(api_key=os.getenv("ELEVENLABS_API_KEY")) tts = ElevenLabsTTSService( api_key=os.getenv("ELEVENLABS_API_KEY", ""), diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py index bbc86d97e..3b3349ba2 100644 --- a/src/pipecat/services/elevenlabs/stt.py +++ b/src/pipecat/services/elevenlabs/stt.py @@ -11,19 +11,43 @@ using segmented audio processing. The service uploads audio files and receives transcription results directly. """ +import base64 import io +import json +from enum import Enum from typing import AsyncGenerator, Optional import aiohttp from loguru import logger from pydantic import BaseModel -from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame -from pipecat.services.stt_service import SegmentedSTTService +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + InterimTranscriptionFrame, + StartFrame, + TranscriptionFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.stt_service import SegmentedSTTService, WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt +try: + from websockets.asyncio.client import connect as websocket_connect + from websockets.protocol import State +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use ElevenLabs Realtime STT, you need to `pip install pipecat-ai[elevenlabs]`." + ) + raise Exception(f"Missing module: {e}") + def language_to_elevenlabs_language(language: Language) -> Optional[str]: """Convert a Language enum to ElevenLabs language code. @@ -329,3 +353,455 @@ class ElevenLabsSTTService(SegmentedSTTService): except Exception as e: logger.error(f"ElevenLabs STT error: {e}") yield ErrorFrame(f"ElevenLabs STT error: {str(e)}") + + +def audio_format_from_sample_rate(sample_rate: int) -> str: + """Get the appropriate audio format string for a given sample rate. + + Args: + sample_rate: The audio sample rate in Hz. + + Returns: + The ElevenLabs audio format string. + """ + match sample_rate: + case 8000: + return "pcm_8000" + case 16000: + return "pcm_16000" + case 22050: + return "pcm_22050" + case 24000: + return "pcm_24000" + case 44100: + return "pcm_44100" + case 48000: + return "pcm_48000" + logger.warning( + f"ElevenLabsRealtimeSTTService: No audio format available for {sample_rate} sample rate, using pcm_16000" + ) + return "pcm_16000" + + +class CommitStrategy(str, Enum): + """Commit strategies for transcript segmentation.""" + + MANUAL = "manual" + VAD = "vad" + + +class ElevenLabsRealtimeSTTService(WebsocketSTTService): + """Speech-to-text service using ElevenLabs' Realtime WebSocket API. + + This service uses ElevenLabs' Realtime Speech-to-Text API to perform transcription + with ultra-low latency. It supports both partial (interim) and committed (final) + transcripts, and can use either manual commit control or automatic Voice Activity + Detection (VAD) for segment boundaries. + + By default, uses manual commit strategy where Pipecat's VAD controls when to + commit transcript segments, providing consistency with other STT services. + """ + + class InputParams(BaseModel): + """Configuration parameters for ElevenLabs Realtime STT API. + + Parameters: + language_code: ISO-639-1 or ISO-639-3 language code. Leave None for auto-detection. + commit_strategy: How to segment speech - manual (Pipecat VAD) or vad (ElevenLabs VAD). + vad_silence_threshold_secs: Seconds of silence before VAD commits (0.3-3.0). + Only used when commit_strategy is VAD. None uses ElevenLabs default. + vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive). + Only used when commit_strategy is VAD. None uses ElevenLabs default. + min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms). + Only used when commit_strategy is VAD. None uses ElevenLabs default. + min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms). + Only used when commit_strategy is VAD. None uses ElevenLabs default. + """ + + language_code: Optional[str] = None + commit_strategy: CommitStrategy = CommitStrategy.MANUAL + vad_silence_threshold_secs: Optional[float] = None + vad_threshold: Optional[float] = None + min_speech_duration_ms: Optional[int] = None + min_silence_duration_ms: Optional[int] = None + + def __init__( + self, + *, + api_key: str, + base_url: str = "api.elevenlabs.io", + model: str = "scribe_v2_realtime", + sample_rate: Optional[int] = None, + params: Optional[InputParams] = None, + **kwargs, + ): + """Initialize the ElevenLabs Realtime STT service. + + Args: + api_key: ElevenLabs API key for authentication. + base_url: Base URL for ElevenLabs WebSocket API. + model: Model ID for transcription. Defaults to "scribe_v2_realtime". + sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate. + params: Configuration parameters for the STT service. + **kwargs: Additional arguments passed to WebsocketSTTService. + """ + super().__init__( + sample_rate=sample_rate, + **kwargs, + ) + + params = params or ElevenLabsRealtimeSTTService.InputParams() + + self._api_key = api_key + self._base_url = base_url + self._model_id = model + self._params = params + self._audio_format = "" # initialized in start() + self._receive_task = None + + def can_generate_metrics(self) -> bool: + """Check if the service can generate processing metrics. + + Returns: + True, as ElevenLabs Realtime STT service supports metrics generation. + """ + return True + + async def set_language(self, language: Language): + """Set the transcription language. + + Args: + language: The language to use for speech-to-text transcription. + + Note: + Changing language requires reconnecting to the WebSocket. + """ + logger.info(f"Switching STT language to: [{language}]") + self._params.language_code = language.value if isinstance(language, Language) else language + # Reconnect with new settings + await self._disconnect() + await self._connect() + + async def set_model(self, model: str): + """Set the STT model. + + Args: + model: The model name to use for transcription. + + Note: + Changing model requires reconnecting to the WebSocket. + """ + await super().set_model(model) + logger.info(f"Switching STT model to: [{model}]") + self._model_id = model + # Reconnect with new settings + await self._disconnect() + await self._connect() + + async def start(self, frame: StartFrame): + """Start the STT service and establish WebSocket connection. + + Args: + frame: Frame indicating service should start. + """ + await super().start(frame) + self._audio_format = audio_format_from_sample_rate(self.sample_rate) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the STT service and close WebSocket connection. + + Args: + frame: Frame indicating service should stop. + """ + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the STT service and close WebSocket connection. + + Args: + frame: Frame indicating service should be cancelled. + """ + await super().cancel(frame) + await self._disconnect() + + async def start_metrics(self): + """Start performance metrics collection for transcription processing.""" + await self.start_ttfb_metrics() + await self.start_processing_metrics() + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle speech events. + + Args: + frame: The frame to process. + direction: Direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, UserStartedSpeakingFrame): + # Start metrics when user starts speaking + await self.start_metrics() + elif isinstance(frame, UserStoppedSpeakingFrame): + # Send commit when user stops speaking (manual commit mode) + if self._params.commit_strategy == CommitStrategy.MANUAL: + if self._websocket and self._websocket.state is State.OPEN: + try: + commit_message = { + "message_type": "input_audio_chunk", + "audio_base_64": "", + "commit": True, + "sample_rate": self.sample_rate, + } + await self._websocket.send(json.dumps(commit_message)) + logger.trace("Sent manual commit to ElevenLabs") + except Exception as e: + logger.warning(f"Failed to send commit: {e}") + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Process audio data for speech-to-text transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + None - transcription results are handled via WebSocket responses. + """ + # Reconnect if connection is closed + if not self._websocket or self._websocket.state is State.CLOSED: + await self._connect() + + if self._websocket and self._websocket.state is State.OPEN: + try: + # Encode audio as base64 + audio_base64 = base64.b64encode(audio).decode("utf-8") + + # Send audio chunk + message = { + "message_type": "input_audio_chunk", + "audio_base_64": audio_base64, + "commit": False, + "sample_rate": self.sample_rate, + } + await self._websocket.send(json.dumps(message)) + except Exception as e: + logger.error(f"Error sending audio: {e}") + yield ErrorFrame(f"ElevenLabs Realtime STT error: {str(e)}") + + yield None + + async def _connect(self): + """Establish WebSocket connection to ElevenLabs Realtime STT.""" + await self._connect_websocket() + + if self._websocket and not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + """Close WebSocket connection and cleanup tasks.""" + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + await self._disconnect_websocket() + + async def _connect_websocket(self): + """Connect to ElevenLabs Realtime STT WebSocket endpoint.""" + try: + if self._websocket and self._websocket.state is State.OPEN: + return + + logger.debug("Connecting to ElevenLabs Realtime STT") + + # Build query parameters + params = [f"model_id={self._model_id}"] + + if self._params.language_code: + params.append(f"language_code={self._params.language_code}") + + params.append(f"encoding={self._audio_format}") + params.append(f"sample_rate={self.sample_rate}") + params.append(f"commit_strategy={self._params.commit_strategy.value}") + + # Add VAD parameters if using VAD commit strategy and values are specified + if self._params.commit_strategy == CommitStrategy.VAD: + if self._params.vad_silence_threshold_secs is not None: + params.append( + f"vad_silence_threshold_secs={self._params.vad_silence_threshold_secs}" + ) + if self._params.vad_threshold is not None: + params.append(f"vad_threshold={self._params.vad_threshold}") + if self._params.min_speech_duration_ms is not None: + params.append(f"min_speech_duration_ms={self._params.min_speech_duration_ms}") + if self._params.min_silence_duration_ms is not None: + params.append(f"min_silence_duration_ms={self._params.min_silence_duration_ms}") + + ws_url = f"wss://{self._base_url}/v1/speech-to-text/realtime?{'&'.join(params)}" + + headers = {"xi-api-key": self._api_key} + + self._websocket = await websocket_connect(ws_url, additional_headers=headers) + await self._call_event_handler("on_connected") + logger.debug("Connected to ElevenLabs Realtime STT") + except Exception as e: + logger.error(f"{self}: unable to connect to ElevenLabs Realtime STT: {e}") + await self.push_error(ErrorFrame(f"Connection error: {str(e)}")) + + async def _disconnect_websocket(self): + """Disconnect from ElevenLabs Realtime STT WebSocket.""" + try: + if self._websocket and self._websocket.state is State.OPEN: + logger.debug("Disconnecting from ElevenLabs Realtime STT") + await self._websocket.close() + except Exception as e: + logger.error(f"{self} error closing websocket: {e}") + finally: + self._websocket = None + await self._call_event_handler("on_disconnected") + + def _get_websocket(self): + """Get the current WebSocket connection. + + Returns: + The WebSocket connection. + + Raises: + Exception: If WebSocket is not connected. + """ + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def _process_messages(self): + """Process incoming WebSocket messages.""" + async for message in self._get_websocket(): + try: + data = json.loads(message) + await self._process_response(data) + except json.JSONDecodeError: + logger.warning(f"Received non-JSON message: {message}") + except Exception as e: + logger.error(f"Error processing message: {e}") + + async def _receive_messages(self): + """Continuously receive and process WebSocket messages.""" + try: + await self._process_messages() + except Exception as e: + logger.warning(f"{self} WebSocket connection closed: {e}") + # Connection closed, will reconnect on next audio chunk + + async def _process_response(self, data: dict): + """Process a response message from ElevenLabs. + + Args: + data: Parsed JSON response data. + """ + message_type = data.get("message_type") + + if message_type == "session_started": + logger.debug(f"ElevenLabs session started: {data}") + + elif message_type == "partial_transcript": + await self._on_partial_transcript(data) + + elif message_type == "committed_transcript": + await self._on_committed_transcript(data) + + elif message_type == "committed_transcript_with_timestamps": + await self._on_committed_transcript_with_timestamps(data) + + elif message_type == "input_error": + error_msg = data.get("error", "Unknown input error") + logger.error(f"ElevenLabs input error: {error_msg}") + await self.push_error(ErrorFrame(f"Input error: {error_msg}")) + + elif message_type in ["auth_error", "quota_exceeded", "transcriber_error", "error"]: + error_msg = data.get("error", data.get("message", "Unknown error")) + logger.error(f"ElevenLabs error ({message_type}): {error_msg}") + await self.push_error(ErrorFrame(f"{message_type}: {error_msg}")) + + else: + logger.debug(f"Unknown message type: {message_type}") + + async def _on_partial_transcript(self, data: dict): + """Handle partial transcript (interim results). + + Args: + data: Partial transcript data. + """ + text = data.get("text", "").strip() + if not text: + return + + await self.stop_ttfb_metrics() + + # Get language if provided + language = data.get("language_code") + + logger.trace(f"Partial transcript: [{text}]") + + await self.push_frame( + InterimTranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + language, + result=data, + ) + ) + + @traced_stt + async def _handle_transcription( + self, transcript: str, is_final: bool, language: Optional[str] = None + ): + """Handle a transcription result with tracing.""" + pass + + async def _on_committed_transcript(self, data: dict): + """Handle committed transcript (final results). + + Args: + data: Committed transcript data. + """ + text = data.get("text", "").strip() + if not text: + return + + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() + + # Get language if provided + language = data.get("language_code") + + logger.debug(f"Committed transcript: [{text}]") + + await self._handle_transcription(text, True, language) + + await self.push_frame( + TranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + language, + result=data, + ) + ) + + async def _on_committed_transcript_with_timestamps(self, data: dict): + """Handle committed transcript with word-level timestamps. + + Args: + data: Committed transcript data with timestamps. + """ + text = data.get("text", "").strip() + if not text: + return + + logger.debug(f"Committed transcript with timestamps: [{text}]") + logger.trace(f"Timestamps: {data.get('words', [])}") + + # This is sent after the committed_transcript, so we don't need to + # push another TranscriptionFrame, but we could use the timestamps + # for additional processing if needed in the future From 8851d18f92c9e45d952d9a77510078d4785475cd Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 13 Nov 2025 09:56:03 -0500 Subject: [PATCH 030/110] Tweak the LLM prompt again to try to fix the issue of LLMs sometimes omitting punctuation in their output. --- examples/foundational/04-transports-small-webrtc.py | 2 +- examples/foundational/04a-transports-daily.py | 2 +- examples/foundational/04b-transports-livekit.py | 2 +- examples/foundational/06-listen-and-respond.py | 2 +- examples/foundational/06a-image-sync.py | 2 +- examples/foundational/07-interruptible-cartesia-http.py | 2 +- examples/foundational/07-interruptible.py | 2 +- examples/foundational/07a-interruptible-speechmatics-vad.py | 2 +- examples/foundational/07a-interruptible-speechmatics.py | 2 +- examples/foundational/07aa-interruptible-soniox.py | 2 +- examples/foundational/07ab-interruptible-inworld-http.py | 2 +- examples/foundational/07ac-interruptible-asyncai-http.py | 2 +- examples/foundational/07ac-interruptible-asyncai.py | 2 +- examples/foundational/07ad-interruptible-aicoustics.py | 2 +- examples/foundational/07ae-interruptible-hume.py | 2 +- examples/foundational/07c-interruptible-deepgram-flux.py | 2 +- examples/foundational/07c-interruptible-deepgram-http.py | 2 +- examples/foundational/07c-interruptible-deepgram-vad.py | 2 +- examples/foundational/07c-interruptible-deepgram.py | 2 +- examples/foundational/07d-interruptible-elevenlabs-http.py | 2 +- examples/foundational/07d-interruptible-elevenlabs.py | 2 +- examples/foundational/07e-interruptible-playht-http.py | 2 +- examples/foundational/07e-interruptible-playht.py | 2 +- examples/foundational/07f-interruptible-azure-http.py | 2 +- examples/foundational/07f-interruptible-azure.py | 2 +- examples/foundational/07g-interruptible-openai.py | 2 +- examples/foundational/07h-interruptible-openpipe.py | 2 +- examples/foundational/07i-interruptible-xtts.py | 2 +- examples/foundational/07j-interruptible-gladia.py | 2 +- examples/foundational/07k-interruptible-lmnt.py | 2 +- examples/foundational/07l-interruptible-groq.py | 2 +- examples/foundational/07m-interruptible-aws.py | 2 +- examples/foundational/07n-interruptible-gemini-image.py | 2 +- examples/foundational/07n-interruptible-gemini.py | 2 +- examples/foundational/07n-interruptible-google-http.py | 2 +- examples/foundational/07n-interruptible-google.py | 2 +- examples/foundational/07o-interruptible-assemblyai.py | 2 +- examples/foundational/07p-interruptible-krisp-viva.py | 2 +- examples/foundational/07p-interruptible-krisp.py | 2 +- examples/foundational/07q-interruptible-rime-http.py | 2 +- examples/foundational/07q-interruptible-rime.py | 2 +- examples/foundational/07r-interruptible-riva-nim.py | 2 +- examples/foundational/07s-interruptible-google-audio-in.py | 2 +- examples/foundational/07t-interruptible-fish.py | 2 +- examples/foundational/07v-interruptible-neuphonic-http.py | 2 +- examples/foundational/07v-interruptible-neuphonic.py | 2 +- examples/foundational/07w-interruptible-fal.py | 2 +- examples/foundational/07x-interruptible-local.py | 2 +- examples/foundational/07y-interruptible-minimax.py | 2 +- examples/foundational/07z-interruptible-sarvam-http.py | 2 +- examples/foundational/07z-interruptible-sarvam.py | 2 +- examples/foundational/08-custom-frame-processor.py | 2 +- examples/foundational/11-sound-effects.py | 2 +- examples/foundational/12-describe-image-openai.py | 2 +- examples/foundational/12a-describe-image-anthropic.py | 2 +- examples/foundational/12b-describe-image-aws.py | 2 +- examples/foundational/12c-describe-image-gemini-flash.py | 2 +- examples/foundational/14-function-calling.py | 2 +- examples/foundational/14c-function-calling-together.py | 2 +- .../foundational/14d-function-calling-anthropic-video.py | 2 +- examples/foundational/14d-function-calling-aws-video.py | 2 +- .../foundational/14d-function-calling-gemini-flash-video.py | 2 +- .../foundational/14d-function-calling-moondream-video.py | 2 +- examples/foundational/14d-function-calling-openai-video.py | 2 +- examples/foundational/14f-function-calling-groq.py | 2 +- examples/foundational/14g-function-calling-grok.py | 2 +- examples/foundational/14h-function-calling-azure.py | 2 +- examples/foundational/14i-function-calling-fireworks.py | 2 +- examples/foundational/14j-function-calling-nim.py | 2 +- examples/foundational/14m-function-calling-openrouter.py | 2 +- examples/foundational/14n-function-calling-perplexity.py | 2 +- examples/foundational/14q-function-calling-qwen.py | 2 +- examples/foundational/14r-function-calling-aws.py | 2 +- examples/foundational/14s-function-calling-sambanova.py | 2 +- examples/foundational/14t-function-calling-direct.py | 2 +- examples/foundational/14u-function-calling-ollama.py | 2 +- examples/foundational/14v-function-calling-openai.py | 2 +- examples/foundational/14w-function-calling-mistral.py | 2 +- examples/foundational/14x-function-calling-openpipe.py | 2 +- examples/foundational/16-gpu-container-local-bot.py | 2 +- examples/foundational/17-detect-user-idle.py | 2 +- examples/foundational/20a-persistent-context-openai.py | 2 +- examples/foundational/20c-persistent-context-anthropic.py | 2 +- examples/foundational/20d-persistent-context-gemini.py | 5 +++-- examples/foundational/21-tavus-transport.py | 2 +- examples/foundational/21a-tavus-video-service.py | 2 +- examples/foundational/22-natural-conversation.py | 2 +- examples/foundational/22b-natural-conversation-proposal.py | 2 +- examples/foundational/22c-natural-conversation-mixed-llms.py | 2 +- .../foundational/22d-natural-conversation-gemini-audio.py | 2 +- examples/foundational/23-bot-background-sound.py | 2 +- examples/foundational/24-stt-mute-filter.py | 2 +- examples/foundational/25-google-audio-in.py | 2 +- examples/foundational/26-gemini-live.py | 2 +- examples/foundational/26d-gemini-live-text.py | 2 +- examples/foundational/26f-gemini-live-files-api.py | 2 +- examples/foundational/26g-gemini-live-groundingMetadata.py | 2 +- examples/foundational/27-simli-layer.py | 2 +- examples/foundational/28-transcription-processor.py | 2 +- examples/foundational/29-turn-tracking-observer.py | 2 +- examples/foundational/30-observer.py | 2 +- examples/foundational/36-user-email-gathering.py | 4 ++-- examples/foundational/38-smart-turn-fal.py | 2 +- examples/foundational/38a-smart-turn-local-coreml.py | 2 +- examples/foundational/38b-smart-turn-local.py | 2 +- examples/foundational/39-mcp-stdio.py | 2 +- examples/foundational/39a-mcp-run-sse.py | 2 +- examples/foundational/39b-multiple-mcp.py | 2 +- examples/foundational/39c-mcp-run-http.py | 2 +- examples/foundational/39d-mcp-run-http-gemini-live.py | 2 +- examples/foundational/42-interruption-config.py | 2 +- examples/foundational/43-heygen-transport.py | 2 +- examples/foundational/43a-heygen-video-service.py | 2 +- examples/foundational/44-voicemail-detection.py | 2 +- examples/foundational/45-before-and-after-events.py | 2 +- examples/foundational/46-video-processing.py | 2 +- examples/foundational/47-sentry-metrics.py | 2 +- examples/foundational/48-service-switcher.py | 2 +- 118 files changed, 121 insertions(+), 120 deletions(-) diff --git a/examples/foundational/04-transports-small-webrtc.py b/examples/foundational/04-transports-small-webrtc.py index 292d5f961..9a622e200 100644 --- a/examples/foundational/04-transports-small-webrtc.py +++ b/examples/foundational/04-transports-small-webrtc.py @@ -77,7 +77,7 @@ async def run_example(webrtc_connection: SmallWebRTCConnection): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/04a-transports-daily.py b/examples/foundational/04a-transports-daily.py index 228905e89..7e5e432ff 100644 --- a/examples/foundational/04a-transports-daily.py +++ b/examples/foundational/04a-transports-daily.py @@ -60,7 +60,7 @@ async def main(): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/04b-transports-livekit.py b/examples/foundational/04b-transports-livekit.py index 51705240f..d2941e2b7 100644 --- a/examples/foundational/04b-transports-livekit.py +++ b/examples/foundational/04b-transports-livekit.py @@ -69,7 +69,7 @@ async def main(): "role": "system", "content": "You are a helpful LLM in a WebRTC call. " "Your goal is to demonstrate your capabilities in a succinct way. " - "Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. " + "Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. " "Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index 638574975..5b1eed538 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -100,7 +100,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index b4ea804eb..e0edf1b36 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -113,7 +113,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07-interruptible-cartesia-http.py b/examples/foundational/07-interruptible-cartesia-http.py index 599cbecc5..299332459 100644 --- a/examples/foundational/07-interruptible-cartesia-http.py +++ b/examples/foundational/07-interruptible-cartesia-http.py @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 13a4738fc..d6699b390 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -70,7 +70,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07a-interruptible-speechmatics-vad.py b/examples/foundational/07a-interruptible-speechmatics-vad.py index 2c42a3e4a..1a58e724f 100644 --- a/examples/foundational/07a-interruptible-speechmatics-vad.py +++ b/examples/foundational/07a-interruptible-speechmatics-vad.py @@ -121,7 +121,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): "content": ( "You are a helpful British assistant called Sarah. " "Your goal is to demonstrate your capabilities in a succinct way. " - "Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. " + "Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. " "Always include punctuation in your responses. " "Give very short replies - do not give longer replies unless strictly necessary. " "Respond to what the user said in a concise, funny, creative and helpful way. " diff --git a/examples/foundational/07a-interruptible-speechmatics.py b/examples/foundational/07a-interruptible-speechmatics.py index 4edb62f7e..558caff0a 100644 --- a/examples/foundational/07a-interruptible-speechmatics.py +++ b/examples/foundational/07a-interruptible-speechmatics.py @@ -111,7 +111,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): "content": ( "You are a helpful British assistant called Sarah. " "Your goal is to demonstrate your capabilities in a succinct way. " - "Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. " + "Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. " "Always include punctuation in your responses. " "Give very short replies - do not give longer replies unless strictly necessary. " "Respond to what the user said in a concise, funny, creative and helpful way. " diff --git a/examples/foundational/07aa-interruptible-soniox.py b/examples/foundational/07aa-interruptible-soniox.py index a5462a4af..b211837a9 100644 --- a/examples/foundational/07aa-interruptible-soniox.py +++ b/examples/foundational/07aa-interruptible-soniox.py @@ -70,7 +70,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07ab-interruptible-inworld-http.py b/examples/foundational/07ab-interruptible-inworld-http.py index d86f1b241..895e42ede 100644 --- a/examples/foundational/07ab-interruptible-inworld-http.py +++ b/examples/foundational/07ab-interruptible-inworld-http.py @@ -81,7 +81,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are very knowledgable about dogs. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are very knowledgable about dogs. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07ac-interruptible-asyncai-http.py b/examples/foundational/07ac-interruptible-asyncai-http.py index da2df63b0..237104bd2 100644 --- a/examples/foundational/07ac-interruptible-asyncai-http.py +++ b/examples/foundational/07ac-interruptible-asyncai-http.py @@ -76,7 +76,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07ac-interruptible-asyncai.py b/examples/foundational/07ac-interruptible-asyncai.py index 08680e569..35d3e89e9 100644 --- a/examples/foundational/07ac-interruptible-asyncai.py +++ b/examples/foundational/07ac-interruptible-asyncai.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07ad-interruptible-aicoustics.py b/examples/foundational/07ad-interruptible-aicoustics.py index 4b5fea5ae..16a246699 100644 --- a/examples/foundational/07ad-interruptible-aicoustics.py +++ b/examples/foundational/07ad-interruptible-aicoustics.py @@ -95,7 +95,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index 2c7a651dc..046f2d4c8 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07c-interruptible-deepgram-flux.py b/examples/foundational/07c-interruptible-deepgram-flux.py index 39fa8fc6a..f5b246acd 100644 --- a/examples/foundational/07c-interruptible-deepgram-flux.py +++ b/examples/foundational/07c-interruptible-deepgram-flux.py @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07c-interruptible-deepgram-http.py b/examples/foundational/07c-interruptible-deepgram-http.py index f614835e2..03375c27a 100644 --- a/examples/foundational/07c-interruptible-deepgram-http.py +++ b/examples/foundational/07c-interruptible-deepgram-http.py @@ -75,7 +75,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07c-interruptible-deepgram-vad.py b/examples/foundational/07c-interruptible-deepgram-vad.py index c17880d4a..22498afe9 100644 --- a/examples/foundational/07c-interruptible-deepgram-vad.py +++ b/examples/foundational/07c-interruptible-deepgram-vad.py @@ -68,7 +68,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07c-interruptible-deepgram.py b/examples/foundational/07c-interruptible-deepgram.py index 17aa4d541..e73711733 100644 --- a/examples/foundational/07c-interruptible-deepgram.py +++ b/examples/foundational/07c-interruptible-deepgram.py @@ -69,7 +69,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07d-interruptible-elevenlabs-http.py b/examples/foundational/07d-interruptible-elevenlabs-http.py index d07e8bb3b..7d3d5c0f8 100644 --- a/examples/foundational/07d-interruptible-elevenlabs-http.py +++ b/examples/foundational/07d-interruptible-elevenlabs-http.py @@ -79,7 +79,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index 2fccccbfa..1f63a469d 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07e-interruptible-playht-http.py b/examples/foundational/07e-interruptible-playht-http.py index e89d5d149..5d6b7ceec 100644 --- a/examples/foundational/07e-interruptible-playht-http.py +++ b/examples/foundational/07e-interruptible-playht-http.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07e-interruptible-playht.py b/examples/foundational/07e-interruptible-playht.py index 2e627d447..f4a23772b 100644 --- a/examples/foundational/07e-interruptible-playht.py +++ b/examples/foundational/07e-interruptible-playht.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07f-interruptible-azure-http.py b/examples/foundational/07f-interruptible-azure-http.py index 9ae460033..0ce19bf48 100644 --- a/examples/foundational/07f-interruptible-azure-http.py +++ b/examples/foundational/07f-interruptible-azure-http.py @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07f-interruptible-azure.py b/examples/foundational/07f-interruptible-azure.py index 61cff1dcb..6d4cf5793 100644 --- a/examples/foundational/07f-interruptible-azure.py +++ b/examples/foundational/07f-interruptible-azure.py @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index fe3afcc54..aa44e5a42 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are very knowledgable about dogs. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are very knowledgable about dogs. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index a7a37dc6f..60565d1f9 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07i-interruptible-xtts.py b/examples/foundational/07i-interruptible-xtts.py index 4936fe165..9ad73c7d2 100644 --- a/examples/foundational/07i-interruptible-xtts.py +++ b/examples/foundational/07i-interruptible-xtts.py @@ -75,7 +75,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index 788a0101d..079967857 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -81,7 +81,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": f"You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": f"You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index f9dfb436b..2d57b28a5 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -68,7 +68,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07l-interruptible-groq.py b/examples/foundational/07l-interruptible-groq.py index 156e5400e..6938a1598 100644 --- a/examples/foundational/07l-interruptible-groq.py +++ b/examples/foundational/07l-interruptible-groq.py @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07m-interruptible-aws.py b/examples/foundational/07m-interruptible-aws.py index 89dd232fe..b53f1f367 100644 --- a/examples/foundational/07m-interruptible-aws.py +++ b/examples/foundational/07m-interruptible-aws.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07n-interruptible-gemini-image.py b/examples/foundational/07n-interruptible-gemini-image.py index ff79064c2..ef37d1fb6 100644 --- a/examples/foundational/07n-interruptible-gemini-image.py +++ b/examples/foundational/07n-interruptible-gemini-image.py @@ -94,7 +94,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07n-interruptible-gemini.py b/examples/foundational/07n-interruptible-gemini.py index 4da14f908..f3f5dff20 100644 --- a/examples/foundational/07n-interruptible-gemini.py +++ b/examples/foundational/07n-interruptible-gemini.py @@ -109,7 +109,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): Feel free to use natural language instructions to control your voice style, tone, pace, and emotion. The TTS system will interpret these instructions and adjust the speech accordingly. - Your output will be converted to audio, so avoid special characters in your answers. Respond to what the user said in a creative and helpful way.""", + Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.""", }, ] diff --git a/examples/foundational/07n-interruptible-google-http.py b/examples/foundational/07n-interruptible-google-http.py index 1fa81354b..2ef65e474 100644 --- a/examples/foundational/07n-interruptible-google-http.py +++ b/examples/foundational/07n-interruptible-google-http.py @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07n-interruptible-google.py b/examples/foundational/07n-interruptible-google.py index 007f3db15..73dd49e78 100644 --- a/examples/foundational/07n-interruptible-google.py +++ b/examples/foundational/07n-interruptible-google.py @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index a3d9d5b56..2a76dbad8 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07p-interruptible-krisp-viva.py b/examples/foundational/07p-interruptible-krisp-viva.py index ac85cce46..c8b374dac 100644 --- a/examples/foundational/07p-interruptible-krisp-viva.py +++ b/examples/foundational/07p-interruptible-krisp-viva.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07p-interruptible-krisp.py b/examples/foundational/07p-interruptible-krisp.py index ab10fa9c3..5dfaaff44 100644 --- a/examples/foundational/07p-interruptible-krisp.py +++ b/examples/foundational/07p-interruptible-krisp.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07q-interruptible-rime-http.py b/examples/foundational/07q-interruptible-rime-http.py index baecdfd3c..d5fa8b710 100644 --- a/examples/foundational/07q-interruptible-rime-http.py +++ b/examples/foundational/07q-interruptible-rime-http.py @@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07q-interruptible-rime.py b/examples/foundational/07q-interruptible-rime.py index 1cbc84246..e66222db9 100644 --- a/examples/foundational/07q-interruptible-rime.py +++ b/examples/foundational/07q-interruptible-rime.py @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py index 629f98f58..a9c1f74fd 100644 --- a/examples/foundational/07r-interruptible-riva-nim.py +++ b/examples/foundational/07r-interruptible-riva-nim.py @@ -68,7 +68,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index d87c6ebc5..67772e40d 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -53,7 +53,7 @@ You are a helpful LLM in a WebRTC call. Your goals are to be helpful and brief i You are expert at transcribing audio to text. You will receive a mixture of audio and text input. When asked to transcribe what the user said, output an exact, word-for-word transcription. -Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. +Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Each time you answer, you should respond in three parts. diff --git a/examples/foundational/07t-interruptible-fish.py b/examples/foundational/07t-interruptible-fish.py index 7e756da15..53ee61dea 100644 --- a/examples/foundational/07t-interruptible-fish.py +++ b/examples/foundational/07t-interruptible-fish.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07v-interruptible-neuphonic-http.py b/examples/foundational/07v-interruptible-neuphonic-http.py index e2b26e90d..6de428d8b 100644 --- a/examples/foundational/07v-interruptible-neuphonic-http.py +++ b/examples/foundational/07v-interruptible-neuphonic-http.py @@ -76,7 +76,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07v-interruptible-neuphonic.py b/examples/foundational/07v-interruptible-neuphonic.py index 7cadececb..b0a49104e 100644 --- a/examples/foundational/07v-interruptible-neuphonic.py +++ b/examples/foundational/07v-interruptible-neuphonic.py @@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07w-interruptible-fal.py b/examples/foundational/07w-interruptible-fal.py index 1c4e3a491..6836f439e 100644 --- a/examples/foundational/07w-interruptible-fal.py +++ b/examples/foundational/07w-interruptible-fal.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07x-interruptible-local.py b/examples/foundational/07x-interruptible-local.py index 9b005712f..ce9c7597d 100644 --- a/examples/foundational/07x-interruptible-local.py +++ b/examples/foundational/07x-interruptible-local.py @@ -56,7 +56,7 @@ async def main(): messages = [ { "role": "system", - "content": "You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07y-interruptible-minimax.py b/examples/foundational/07y-interruptible-minimax.py index 8e7e38723..6a5d33887 100644 --- a/examples/foundational/07y-interruptible-minimax.py +++ b/examples/foundational/07y-interruptible-minimax.py @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07z-interruptible-sarvam-http.py b/examples/foundational/07z-interruptible-sarvam-http.py index 634f0a779..73239167d 100644 --- a/examples/foundational/07z-interruptible-sarvam-http.py +++ b/examples/foundational/07z-interruptible-sarvam-http.py @@ -80,7 +80,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/07z-interruptible-sarvam.py b/examples/foundational/07z-interruptible-sarvam.py index 9590514fe..41418049f 100644 --- a/examples/foundational/07z-interruptible-sarvam.py +++ b/examples/foundational/07z-interruptible-sarvam.py @@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/08-custom-frame-processor.py b/examples/foundational/08-custom-frame-processor.py index 6c25af0c3..72b16abe6 100644 --- a/examples/foundational/08-custom-frame-processor.py +++ b/examples/foundational/08-custom-frame-processor.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index 9eae5944f..a15bae147 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -121,7 +121,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/12-describe-image-openai.py b/examples/foundational/12-describe-image-openai.py index 3322c3572..8c72075e8 100644 --- a/examples/foundational/12-describe-image-openai.py +++ b/examples/foundational/12-describe-image-openai.py @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. You are also able to describe images.", }, ] diff --git a/examples/foundational/12a-describe-image-anthropic.py b/examples/foundational/12a-describe-image-anthropic.py index e2e1ccbad..6a9891712 100644 --- a/examples/foundational/12a-describe-image-anthropic.py +++ b/examples/foundational/12a-describe-image-anthropic.py @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. You are also able to describe images.", }, ] diff --git a/examples/foundational/12b-describe-image-aws.py b/examples/foundational/12b-describe-image-aws.py index 6c7f9c3fa..441c49cfd 100644 --- a/examples/foundational/12b-describe-image-aws.py +++ b/examples/foundational/12b-describe-image-aws.py @@ -73,7 +73,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. You are also able to describe images.", }, ] diff --git a/examples/foundational/12c-describe-image-gemini-flash.py b/examples/foundational/12c-describe-image-gemini-flash.py index 8a388efe6..919bf3553 100644 --- a/examples/foundational/12c-describe-image-gemini-flash.py +++ b/examples/foundational/12c-describe-image-gemini-flash.py @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. You are also able to describe images.", }, ] diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index dbab11afc..3f30e3389 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -120,7 +120,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index 89fe81ac8..d46b2afdb 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -106,7 +106,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14d-function-calling-anthropic-video.py b/examples/foundational/14d-function-calling-anthropic-video.py index f3b4ba36d..9f8dbcb76 100644 --- a/examples/foundational/14d-function-calling-anthropic-video.py +++ b/examples/foundational/14d-function-calling-anthropic-video.py @@ -119,7 +119,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", }, ] diff --git a/examples/foundational/14d-function-calling-aws-video.py b/examples/foundational/14d-function-calling-aws-video.py index 0a9a1824f..f807e5bff 100644 --- a/examples/foundational/14d-function-calling-aws-video.py +++ b/examples/foundational/14d-function-calling-aws-video.py @@ -126,7 +126,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", }, ] diff --git a/examples/foundational/14d-function-calling-gemini-flash-video.py b/examples/foundational/14d-function-calling-gemini-flash-video.py index d166777c9..5af3bc6b0 100644 --- a/examples/foundational/14d-function-calling-gemini-flash-video.py +++ b/examples/foundational/14d-function-calling-gemini-flash-video.py @@ -119,7 +119,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", }, ] diff --git a/examples/foundational/14d-function-calling-moondream-video.py b/examples/foundational/14d-function-calling-moondream-video.py index 51f567f4a..6aeb2b892 100644 --- a/examples/foundational/14d-function-calling-moondream-video.py +++ b/examples/foundational/14d-function-calling-moondream-video.py @@ -120,7 +120,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", }, ] diff --git a/examples/foundational/14d-function-calling-openai-video.py b/examples/foundational/14d-function-calling-openai-video.py index 272ea47b7..f0d36bca4 100644 --- a/examples/foundational/14d-function-calling-openai-video.py +++ b/examples/foundational/14d-function-calling-openai-video.py @@ -119,7 +119,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", }, ] diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 13f121ab4..53eb2de75 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -104,7 +104,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 9a75a513e..ffd5ad947 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -99,7 +99,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index 36f2268e4..71c3286e8 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -107,7 +107,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index ee4732d43..235dcf8cc 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. Start by saying hello.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. Start by saying hello.", }, ] diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index d02e89e85..97841f1a1 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -112,7 +112,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): {"role": "system", "content": "/no_think"}, { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index a1ecc91ff..ea16d503a 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -107,7 +107,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index 01e91e562..2dac6250e 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "user", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way, but try to be brief.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way, but try to be brief.", }, ] diff --git a/examples/foundational/14q-function-calling-qwen.py b/examples/foundational/14q-function-calling-qwen.py index 69e312cfc..f49c0631c 100644 --- a/examples/foundational/14q-function-calling-qwen.py +++ b/examples/foundational/14q-function-calling-qwen.py @@ -105,7 +105,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14r-function-calling-aws.py b/examples/foundational/14r-function-calling-aws.py index bc4b9ffa9..5e005086c 100644 --- a/examples/foundational/14r-function-calling-aws.py +++ b/examples/foundational/14r-function-calling-aws.py @@ -120,7 +120,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14s-function-calling-sambanova.py b/examples/foundational/14s-function-calling-sambanova.py index 6d16daade..dae1531bc 100644 --- a/examples/foundational/14s-function-calling-sambanova.py +++ b/examples/foundational/14s-function-calling-sambanova.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14t-function-calling-direct.py b/examples/foundational/14t-function-calling-direct.py index eedf74fa7..feae09083 100644 --- a/examples/foundational/14t-function-calling-direct.py +++ b/examples/foundational/14t-function-calling-direct.py @@ -106,7 +106,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14u-function-calling-ollama.py b/examples/foundational/14u-function-calling-ollama.py index 250da11c2..f60af9f64 100644 --- a/examples/foundational/14u-function-calling-ollama.py +++ b/examples/foundational/14u-function-calling-ollama.py @@ -122,7 +122,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14v-function-calling-openai.py b/examples/foundational/14v-function-calling-openai.py index cf3039dde..06c3e2abd 100644 --- a/examples/foundational/14v-function-calling-openai.py +++ b/examples/foundational/14v-function-calling-openai.py @@ -128,7 +128,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14w-function-calling-mistral.py b/examples/foundational/14w-function-calling-mistral.py index e1fd1b2eb..82a48f6f6 100644 --- a/examples/foundational/14w-function-calling-mistral.py +++ b/examples/foundational/14w-function-calling-mistral.py @@ -116,7 +116,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/14x-function-calling-openpipe.py b/examples/foundational/14x-function-calling-openpipe.py index 78ff796c9..ac918a0ad 100644 --- a/examples/foundational/14x-function-calling-openpipe.py +++ b/examples/foundational/14x-function-calling-openpipe.py @@ -126,7 +126,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/16-gpu-container-local-bot.py b/examples/foundational/16-gpu-container-local-bot.py index 39ad7d345..1e40a33f5 100644 --- a/examples/foundational/16-gpu-container-local-bot.py +++ b/examples/foundational/16-gpu-container-local-bot.py @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 6d0a94460..e9671e145 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 4ecda106d..1a885b1fd 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -98,7 +98,7 @@ async def load_conversation(params: FunctionCallParams): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 77dc3627c..5584d525b 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -100,7 +100,7 @@ async def load_conversation(params: FunctionCallParams): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a succinct, creative and helpful way. Prefer responses that are one sentence long unless you are asked for a longer or more detailed response.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a succinct, creative and helpful way. Prefer responses that are one sentence long unless you are asked for a longer or more detailed response.", }, {"role": "user", "content": "Start the call by saying the word 'hello'. Say only that word."}, # {"role": "user", "content": ""}, diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index b32c2fd5b..e618b7d10 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -121,8 +121,9 @@ messages = [ { "role": "system", "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your -capabilities in a succinct way. Your output will be converted to audio so don't include special -characters in your answers. Respond to what the user said in a creative and helpful way. +capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that +can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative +and helpful way. You have several tools you can use to help you. diff --git a/examples/foundational/21-tavus-transport.py b/examples/foundational/21-tavus-transport.py index 0e68c8507..b6643d668 100644 --- a/examples/foundational/21-tavus-transport.py +++ b/examples/foundational/21-tavus-transport.py @@ -61,7 +61,7 @@ async def main(): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/21a-tavus-video-service.py b/examples/foundational/21a-tavus-video-service.py index d2421eae4..b35b315bb 100644 --- a/examples/foundational/21a-tavus-video-service.py +++ b/examples/foundational/21a-tavus-video-service.py @@ -80,7 +80,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 2ccadbd2b..f95a473df 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -142,7 +142,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 7e0c85284..935676f9b 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -327,7 +327,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index db0740a37..cdcd21289 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -258,7 +258,7 @@ Output: YES Output: NO """ -conversational_system_message = """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. +conversational_system_message = """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. Please be very concise in your responses. Unless you are explicitly asked to do otherwise, give me the shortest complete answer possible without unnecessary elaboration. Generally you should answer with a single sentence. """ diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 60ea7d381..7a7155297 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -340,7 +340,7 @@ Output: NO conversation_system_instruction = """You are a helpful assistant participating in a voice converation. -Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way. +Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. If you know that a number string is a phone number from the context of the conversation, write it as a phone number. For example 210-333-4567. diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index e769c9b2a..6a54b43e7 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -90,7 +90,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/24-stt-mute-filter.py b/examples/foundational/24-stt-mute-filter.py index 6c4804c37..7793b3dc0 100644 --- a/examples/foundational/24-stt-mute-filter.py +++ b/examples/foundational/24-stt-mute-filter.py @@ -111,7 +111,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful assistant who can check the weather. Always check the weather when a location is mentioned. Respond concisely and naturally. Your output will be converted to audio so use only simple words and punctuation.", + "content": "You are a helpful assistant who can check the weather. Always check the weather when a location is mentioned. Respond concisely and naturally. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points.", }, ] diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index 0e1413cd1..59a16bced 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -45,7 +45,7 @@ load_dotenv(override=True) # conversation_system_message = """ You are a helpful LLM in a WebRTC call. Your goals are to be helpful and brief in your responses. Respond with one or two sentences at most, unless you are asked to -respond at more length. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. +respond at more length. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. """ # diff --git a/examples/foundational/26-gemini-live.py b/examples/foundational/26-gemini-live.py index 924c3628b..fdabfb0ee 100644 --- a/examples/foundational/26-gemini-live.py +++ b/examples/foundational/26-gemini-live.py @@ -61,7 +61,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): system_instruction = f""" You are a helpful AI assistant. Your goal is to demonstrate your capabilities in a helpful and engaging way. - Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. + Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. """ diff --git a/examples/foundational/26d-gemini-live-text.py b/examples/foundational/26d-gemini-live-text.py index 111f5a6cf..3562d8f2c 100644 --- a/examples/foundational/26d-gemini-live-text.py +++ b/examples/foundational/26d-gemini-live-text.py @@ -38,7 +38,7 @@ SYSTEM_INSTRUCTION = f""" Your goal is to demonstrate your capabilities in a succinct way. -Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. +Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. Keep your responses brief. One or two sentences at most. """ diff --git a/examples/foundational/26f-gemini-live-files-api.py b/examples/foundational/26f-gemini-live-files-api.py index 855cd151c..bb9791a05 100644 --- a/examples/foundational/26f-gemini-live-files-api.py +++ b/examples/foundational/26f-gemini-live-files-api.py @@ -105,7 +105,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - Answer questions about what's in the document - Use the information from the document in our conversation - Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. + Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Be friendly and demonstrate your ability to work with the uploaded file. """ diff --git a/examples/foundational/26g-gemini-live-groundingMetadata.py b/examples/foundational/26g-gemini-live-groundingMetadata.py index 468fcd678..c05f63dad 100644 --- a/examples/foundational/26g-gemini-live-groundingMetadata.py +++ b/examples/foundational/26g-gemini-live-groundingMetadata.py @@ -63,7 +63,7 @@ You should use Google Search for: Always be proactive about using search when the user asks about anything that could benefit from real-time information. -Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. +Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way, always using search for current information. """ diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index c2c3f35ce..bf2d56ca0 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/28-transcription-processor.py b/examples/foundational/28-transcription-processor.py index 21d800198..8258763be 100644 --- a/examples/foundational/28-transcription-processor.py +++ b/examples/foundational/28-transcription-processor.py @@ -134,7 +134,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative, helpful, and brief way. Say hello.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative, helpful, and brief way. Say hello.", }, ] diff --git a/examples/foundational/29-turn-tracking-observer.py b/examples/foundational/29-turn-tracking-observer.py index 6656ed83a..3965b2953 100644 --- a/examples/foundational/29-turn-tracking-observer.py +++ b/examples/foundational/29-turn-tracking-observer.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index 6e02aee37..c41177df2 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -119,7 +119,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/36-user-email-gathering.py b/examples/foundational/36-user-email-gathering.py index e69f6bee4..3ca6b01a4 100644 --- a/examples/foundational/36-user-email-gathering.py +++ b/examples/foundational/36-user-email-gathering.py @@ -109,9 +109,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): { "role": "system", # Cartesia - "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. If the user provides one or more email addresses confirm them with the user. Enclose all emails with tags, for example a@a.com.", + "content": "You need to gather a valid email or emails from the user. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. If the user provides one or more email addresses confirm them with the user. Enclose all emails with tags, for example a@a.com.", # Rime spell() - # "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. If the user provides one or more email addresses confirm them with the user. Enclose all emails with spell(), for example spell(a@a.com).", + # "content": "You need to gather a valid email or emails from the user. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. If the user provides one or more email addresses confirm them with the user. Enclose all emails with spell(), for example spell(a@a.com).", }, ] diff --git a/examples/foundational/38-smart-turn-fal.py b/examples/foundational/38-smart-turn-fal.py index 7d01f41f9..c18c7e1c6 100644 --- a/examples/foundational/38-smart-turn-fal.py +++ b/examples/foundational/38-smart-turn-fal.py @@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/38a-smart-turn-local-coreml.py b/examples/foundational/38a-smart-turn-local-coreml.py index ca13be6f2..122cfb463 100644 --- a/examples/foundational/38a-smart-turn-local-coreml.py +++ b/examples/foundational/38a-smart-turn-local-coreml.py @@ -94,7 +94,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/38b-smart-turn-local.py b/examples/foundational/38b-smart-turn-local.py index d00e0ecb3..0f77d73b9 100644 --- a/examples/foundational/38b-smart-turn-local.py +++ b/examples/foundational/38b-smart-turn-local.py @@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/39-mcp-stdio.py b/examples/foundational/39-mcp-stdio.py index f707866b9..b88ee30b1 100644 --- a/examples/foundational/39-mcp-stdio.py +++ b/examples/foundational/39-mcp-stdio.py @@ -158,7 +158,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. - Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. + Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. Don't overexplain what you are doing. Just respond with short sentences when you are carrying out tool calls. diff --git a/examples/foundational/39a-mcp-run-sse.py b/examples/foundational/39a-mcp-run-sse.py index 90de1b91e..328af6ce4 100644 --- a/examples/foundational/39a-mcp-run-sse.py +++ b/examples/foundational/39a-mcp-run-sse.py @@ -90,7 +90,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. You have access to a number of tools provided by mcp.run. Use any and all tools to help users. - Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. + Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. When asked for today's date, use 'https://www.datetoday.net/'. Don't overexplain what you are doing. diff --git a/examples/foundational/39b-multiple-mcp.py b/examples/foundational/39b-multiple-mcp.py index 21e5bff60..dad059208 100644 --- a/examples/foundational/39b-multiple-mcp.py +++ b/examples/foundational/39b-multiple-mcp.py @@ -136,7 +136,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. - Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. + Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. Don't overexplain what you are doing. Just respond with short sentences when you are carrying out tool calls. diff --git a/examples/foundational/39c-mcp-run-http.py b/examples/foundational/39c-mcp-run-http.py index 8e704aaa6..4a94c328f 100644 --- a/examples/foundational/39c-mcp-run-http.py +++ b/examples/foundational/39c-mcp-run-http.py @@ -96,7 +96,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): You are a helpful LLM in a WebRTC call. Your goal is to answer questions about the user's GitHub repositories and account. You have access to a number of tools provided by Github. Use any and all tools to help users. - Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. + Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Don't overexplain what you are doing. Just respond with short sentences when you are carrying out tool calls. """ diff --git a/examples/foundational/39d-mcp-run-http-gemini-live.py b/examples/foundational/39d-mcp-run-http-gemini-live.py index ff2c7649a..101559266 100644 --- a/examples/foundational/39d-mcp-run-http-gemini-live.py +++ b/examples/foundational/39d-mcp-run-http-gemini-live.py @@ -94,7 +94,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): You are a helpful LLM in a WebRTC call. Your goal is to answer questions about the user's GitHub repositories and account. You have access to a number of tools provided by Github. Use any and all tools to help users. - Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. + Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Don't overexplain what you are doing. Just respond with short sentences when you are carrying out tool calls. """ diff --git a/examples/foundational/42-interruption-config.py b/examples/foundational/42-interruption-config.py index 2306ad936..9c312be03 100644 --- a/examples/foundational/42-interruption-config.py +++ b/examples/foundational/42-interruption-config.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/43-heygen-transport.py b/examples/foundational/43-heygen-transport.py index efccbd808..859206cce 100644 --- a/examples/foundational/43-heygen-transport.py +++ b/examples/foundational/43-heygen-transport.py @@ -60,7 +60,7 @@ async def main(): messages = [ { "role": "system", - "content": "You are a helpful assistant. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Be succinct and respond to what the user said in a creative and helpful way.", + "content": "You are a helpful assistant. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Be succinct and respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/43a-heygen-video-service.py b/examples/foundational/43a-heygen-video-service.py index 5f9ddce21..d4c409063 100644 --- a/examples/foundational/43a-heygen-video-service.py +++ b/examples/foundational/43a-heygen-video-service.py @@ -83,7 +83,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful assistant. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Be succinct and respond to what the user said in a creative and helpful way.", + "content": "You are a helpful assistant. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Be succinct and respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index fb612f877..711c2591f 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -74,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/45-before-and-after-events.py b/examples/foundational/45-before-and-after-events.py index b6ad5d807..1cffd533c 100644 --- a/examples/foundational/45-before-and-after-events.py +++ b/examples/foundational/45-before-and-after-events.py @@ -82,7 +82,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/46-video-processing.py b/examples/foundational/46-video-processing.py index c6311db3b..36075d343 100644 --- a/examples/foundational/46-video-processing.py +++ b/examples/foundational/46-video-processing.py @@ -89,7 +89,7 @@ SYSTEM_INSTRUCTION = f""" Your goal is to demonstrate your capabilities in a succinct way. -Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. +Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. Keep your responses brief. One or two sentences at most. """ diff --git a/examples/foundational/47-sentry-metrics.py b/examples/foundational/47-sentry-metrics.py index 958557f81..2f7369349 100644 --- a/examples/foundational/47-sentry-metrics.py +++ b/examples/foundational/47-sentry-metrics.py @@ -85,7 +85,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] diff --git a/examples/foundational/48-service-switcher.py b/examples/foundational/48-service-switcher.py index c13ff7273..cb2757611 100644 --- a/examples/foundational/48-service-switcher.py +++ b/examples/foundational/48-service-switcher.py @@ -129,7 +129,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers other than basic punctuation. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] tools = ToolsSchema(standard_tools=[weather_function, get_restaurant_recommendation]) From edbf96b3c5a17dd22698044bfeaff4d5784cc3ca Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 12 Nov 2025 09:43:52 -0500 Subject: [PATCH 031/110] Update GeminiTTSService for streaming, other Google TTS improvements --- CHANGELOG.md | 15 + .../foundational/07n-interruptible-gemini.py | 50 +- src/pipecat/services/google/tts.py | 769 ++++++++++++------ src/pipecat/transcriptions/language.py | 19 + 4 files changed, 580 insertions(+), 273 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de9cf3328..bb7223b7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added Hindi support for Rime TTS services. +- Updated `GeminiTTSService` to use Google Cloud Text-to-Speech streaming API + instead of the deprecated Gemini API. Now uses `credentials` / + `credentials_path` for authentication. The `api_key` parameter is deprecated. + Also, added support for `prompt` parameter for style instructions and + expressive markup tags. Significantly improved latency with streaming + synthesis. + +- Updated language mappings for the Google and Gemini TTS services to match + official documentation. + +### Deprecated + +- The `api_key` parameter in `GeminiTTSService` is deprecated. Use + `credentials` or `credentials_path` instead for Google Cloud authentication. + ### Fixed - Fixed subtle issue of assistant context messages ending up with double spaces diff --git a/examples/foundational/07n-interruptible-gemini.py b/examples/foundational/07n-interruptible-gemini.py index 4da14f908..3a244a128 100644 --- a/examples/foundational/07n-interruptible-gemini.py +++ b/examples/foundational/07n-interruptible-gemini.py @@ -4,24 +4,6 @@ # SPDX-License-Identifier: BSD 2-Clause License # -""" -A conversational AI bot using Gemini for both LLM and TTS. - -This example demonstrates how to use Gemini's TTS capabilities with the new -GeminiTTSService, which uses Gemini's TTS-specific models instead of Google Cloud TTS. - -Features showcased: -- Gemini LLM for conversation -- Gemini TTS with natural voice control -- Support for different voice personalities -- Style and tone control through natural language prompts - -Run with: - python examples/foundational/gemini-tts.py - -Make sure to set your environment variables: - export GOOGLE_API_KEY=your_api_key_here -""" import os @@ -84,10 +66,13 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) tts = GeminiTTSService( - api_key=os.getenv("GOOGLE_API_KEY"), - model="gemini-2.5-flash-preview-tts", # TTS-specific model + credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + model="gemini-2.5-flash-tts", voice_id="Charon", - params=GeminiTTSService.InputParams(language=Language.EN_US), + params=GeminiTTSService.InputParams( + language=Language.EN_US, + prompt="You are a helpful AI assistant. Speak in a natural, conversational tone.", + ), ) llm = GoogleLLMService( @@ -101,13 +86,20 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): "role": "system", "content": """You are a helpful AI assistant in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. - IMPORTANT: Since you're using Gemini TTS which supports natural voice control, you can include speaking instructions in your responses. For example: - - "Say cheerfully: Welcome to our conversation!" - - "Read this in a calm, professional tone: Here are the details you requested." - - "Speak in an excited whisper: I have some great news to share!" - - "Say slowly and clearly: Let me explain this step by step." + IMPORTANT: You're using Gemini TTS which supports expressive markup tags. You can use these tags in your responses: + - [sigh] - Insert a sigh sound + - [laughing] - Insert a laugh + - [uhm] - Insert a hesitation sound + - [whispering] - Speak the next part in a whisper + - [shouting] - Speak the next part louder + - [extremely fast] - Speak the next part very quickly + - [short pause], [medium pause], [long pause] - Add pauses for dramatic effect - Feel free to use natural language instructions to control your voice style, tone, pace, and emotion. The TTS system will interpret these instructions and adjust the speech accordingly. + Examples: + - "Well [sigh] that's a tricky question." + - "[laughing] That's a great joke!" + - "[whispering] Let me tell you a secret." + - "The answer is... [long pause] ...42!" Your output will be converted to audio, so avoid special characters in your answers. Respond to what the user said in a creative and helpful way.""", }, @@ -140,11 +132,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") - # Kick off the conversation with a styled introduction + # Kick off the conversation messages.append( { "role": "system", - "content": "Say cheerfully and warmly: Hello! I'm your AI assistant powered by Gemini's new TTS technology. I can speak with different voices, tones, and styles. How can I help you today?", + "content": "Hello! I'm your AI assistant. I can help you with a variety of tasks. What would you like to know?", } ) await task.queue_frames([LLMRunFrame()]) diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index bd3dbc203..b20532676 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -16,6 +16,7 @@ for natural voice control and multi-speaker conversations. import json import os +import warnings from pipecat.utils.tracing.service_decorators import traced_tts @@ -51,19 +52,13 @@ except ModuleNotFoundError as e: ) raise Exception(f"Missing module: {e}") -try: - from google import genai - from google.genai import types - -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error("In order to use Gemini TTS, you need to `pip install pipecat-ai[google]`.") - raise Exception(f"Missing module: {e}") - def language_to_google_tts_language(language: Language) -> Optional[str]: """Convert a Language enum to Google TTS language code. + Source: + https://docs.cloud.google.com/text-to-speech/docs/chirp3-hd + Args: language: The Language enum value to convert. @@ -71,9 +66,6 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: The corresponding Google TTS language code, or None if not supported. """ LANGUAGE_MAP = { - # Afrikaans - Language.AF: "af-ZA", - Language.AF_ZA: "af-ZA", # Arabic Language.AR: "ar-XA", # Bengali @@ -82,14 +74,9 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: # Bulgarian Language.BG: "bg-BG", Language.BG_BG: "bg-BG", - # Catalan - Language.CA: "ca-ES", - Language.CA_ES: "ca-ES", - # Chinese (Mandarin and Cantonese) - Language.ZH: "cmn-CN", - Language.ZH_CN: "cmn-CN", - Language.ZH_TW: "cmn-TW", - Language.ZH_HK: "yue-HK", + # Croatian + Language.HR: "hr-HR", + Language.HR_HR: "hr-HR", # Czech Language.CS: "cs-CZ", Language.CS_CZ: "cs-CZ", @@ -109,9 +96,6 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: # Estonian Language.ET: "et-EE", Language.ET_EE: "et-EE", - # Filipino - Language.FIL: "fil-PH", - Language.FIL_PH: "fil-PH", # Finnish Language.FI: "fi-FI", Language.FI_FI: "fi-FI", @@ -119,9 +103,6 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: Language.FR: "fr-FR", Language.FR_CA: "fr-CA", Language.FR_FR: "fr-FR", - # Galician - Language.GL: "gl-ES", - Language.GL_ES: "gl-ES", # German Language.DE: "de-DE", Language.DE_DE: "de-DE", @@ -140,9 +121,6 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: # Hungarian Language.HU: "hu-HU", Language.HU_HU: "hu-HU", - # Icelandic - Language.IS: "is-IS", - Language.IS_IS: "is-IS", # Indonesian Language.ID: "id-ID", Language.ID_ID: "id-ID", @@ -164,12 +142,12 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: # Lithuanian Language.LT: "lt-LT", Language.LT_LT: "lt-LT", - # Malay - Language.MS: "ms-MY", - Language.MS_MY: "ms-MY", # Malayalam Language.ML: "ml-IN", Language.ML_IN: "ml-IN", + # Chinese (Mandarin) + Language.ZH: "cmn-CN", + Language.ZH_CN: "cmn-CN", # Marathi Language.MR: "mr-IN", Language.MR_IN: "mr-IN", @@ -181,12 +159,8 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: Language.PL: "pl-PL", Language.PL_PL: "pl-PL", # Portuguese - Language.PT: "pt-PT", + Language.PT: "pt-BR", Language.PT_BR: "pt-BR", - Language.PT_PT: "pt-PT", - # Punjabi - Language.PA: "pa-IN", - Language.PA_IN: "pa-IN", # Romanian Language.RO: "ro-RO", Language.RO_RO: "ro-RO", @@ -199,10 +173,16 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: # Slovak Language.SK: "sk-SK", Language.SK_SK: "sk-SK", + # Slovenian + Language.SL: "sl-SI", + Language.SL_SI: "sl-SI", # Spanish Language.ES: "es-ES", Language.ES_ES: "es-ES", Language.ES_US: "es-US", + # Swahili + Language.SW: "sw-KE", + Language.SW_KE: "sw-KE", # Swedish Language.SV: "sv-SE", Language.SV_SE: "sv-SE", @@ -221,6 +201,9 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: # Ukrainian Language.UK: "uk-UA", Language.UK_UA: "uk-UA", + # Urdu + Language.UR: "ur-IN", + Language.UR_IN: "ur-IN", # Vietnamese Language.VI: "vi-VN", Language.VI_VN: "vi-VN", @@ -229,6 +212,267 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +def language_to_gemini_tts_language(language: Language) -> Optional[str]: + """Convert a Language enum to Gemini TTS language code. + + Source: + https://docs.cloud.google.com/text-to-speech/docs/gemini-tts#available_languages + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Gemini TTS language code, or None if not supported. + """ + LANGUAGE_MAP = { + # Afrikaans (Preview) + Language.AF: "af-ZA", + Language.AF_ZA: "af-ZA", + # Albanian (Preview) + Language.SQ: "sq-AL", + Language.SQ_AL: "sq-AL", + # Amharic (Preview) + Language.AM: "am-ET", + Language.AM_ET: "am-ET", + # Arabic + Language.AR: "ar-EG", # GA: Egypt + Language.AR_EG: "ar-EG", + Language.AR_001: "ar-001", # Preview: World + # Armenian (Preview) + Language.HY: "hy-AM", + Language.HY_AM: "hy-AM", + # Azerbaijani (Preview) + Language.AZ: "az-AZ", + Language.AZ_AZ: "az-AZ", + # Basque (Preview) + Language.EU: "eu-ES", + Language.EU_ES: "eu-ES", + # Belarusian (Preview) + Language.BE: "be-BY", + Language.BE_BY: "be-BY", + # Bengali (GA) + Language.BN: "bn-BD", + Language.BN_BD: "bn-BD", + # Bulgarian (Preview) + Language.BG: "bg-BG", + Language.BG_BG: "bg-BG", + # Burmese (Preview) + Language.MY: "my-MM", + Language.MY_MM: "my-MM", + # Catalan (Preview) + Language.CA: "ca-ES", + Language.CA_ES: "ca-ES", + # Cebuano (Preview) + Language.CEB: "ceb-PH", + Language.CEB_PH: "ceb-PH", + # Chinese (Mandarin) + Language.ZH: "cmn-CN", # Preview + Language.ZH_CN: "cmn-CN", + Language.ZH_TW: "cmn-TW", # Preview + # Croatian (Preview) + Language.HR: "hr-HR", + Language.HR_HR: "hr-HR", + # Czech (Preview) + Language.CS: "cs-CZ", + Language.CS_CZ: "cs-CZ", + # Danish (Preview) + Language.DA: "da-DK", + Language.DA_DK: "da-DK", + # Dutch (GA) + Language.NL: "nl-NL", + Language.NL_NL: "nl-NL", + # English + Language.EN: "en-US", # GA + Language.EN_US: "en-US", + Language.EN_AU: "en-AU", # Preview + Language.EN_GB: "en-GB", # Preview + Language.EN_IN: "en-IN", # GA + # Estonian (Preview) + Language.ET: "et-EE", + Language.ET_EE: "et-EE", + # Filipino (Preview) + Language.FIL: "fil-PH", + Language.FIL_PH: "fil-PH", + # Finnish (Preview) + Language.FI: "fi-FI", + Language.FI_FI: "fi-FI", + # French + Language.FR: "fr-FR", # GA + Language.FR_FR: "fr-FR", + Language.FR_CA: "fr-CA", # Preview + # Galician (Preview) + Language.GL: "gl-ES", + Language.GL_ES: "gl-ES", + # Georgian (Preview) + Language.KA: "ka-GE", + Language.KA_GE: "ka-GE", + # German (GA) + Language.DE: "de-DE", + Language.DE_DE: "de-DE", + # Greek (Preview) + Language.EL: "el-GR", + Language.EL_GR: "el-GR", + # Gujarati (Preview) + Language.GU: "gu-IN", + Language.GU_IN: "gu-IN", + # Haitian Creole (Preview) + Language.HT: "ht-HT", + Language.HT_HT: "ht-HT", + # Hebrew (Preview) + Language.HE: "he-IL", + Language.HE_IL: "he-IL", + # Hindi (GA) + Language.HI: "hi-IN", + Language.HI_IN: "hi-IN", + # Hungarian (Preview) + Language.HU: "hu-HU", + Language.HU_HU: "hu-HU", + # Icelandic (Preview) + Language.IS: "is-IS", + Language.IS_IS: "is-IS", + # Indonesian (GA) + Language.ID: "id-ID", + Language.ID_ID: "id-ID", + # Italian (GA) + Language.IT: "it-IT", + Language.IT_IT: "it-IT", + # Japanese (GA) + Language.JA: "ja-JP", + Language.JA_JP: "ja-JP", + # Javanese (Preview) + Language.JV: "jv-JV", + Language.JV_JV: "jv-JV", + # Kannada (Preview) + Language.KN: "kn-IN", + Language.KN_IN: "kn-IN", + # Konkani (Preview) + Language.KOK: "kok-IN", + Language.KOK_IN: "kok-IN", + # Korean (GA) + Language.KO: "ko-KR", + Language.KO_KR: "ko-KR", + # Lao (Preview) + Language.LO: "lo-LA", + Language.LO_LA: "lo-LA", + # Latin (Preview) + Language.LA: "la-VA", + Language.LA_VA: "la-VA", + # Latvian (Preview) + Language.LV: "lv-LV", + Language.LV_LV: "lv-LV", + # Lithuanian (Preview) + Language.LT: "lt-LT", + Language.LT_LT: "lt-LT", + # Luxembourgish (Preview) + Language.LB: "lb-LU", + Language.LB_LU: "lb-LU", + # Macedonian (Preview) + Language.MK: "mk-MK", + Language.MK_MK: "mk-MK", + # Maithili (Preview) + Language.MAI: "mai-IN", + Language.MAI_IN: "mai-IN", + # Malagasy (Preview) + Language.MG: "mg-MG", + Language.MG_MG: "mg-MG", + # Malay (Preview) + Language.MS: "ms-MY", + Language.MS_MY: "ms-MY", + # Malayalam (Preview) + Language.ML: "ml-IN", + Language.ML_IN: "ml-IN", + # Marathi (GA) + Language.MR: "mr-IN", + Language.MR_IN: "mr-IN", + # Mongolian (Preview) + Language.MN: "mn-MN", + Language.MN_MN: "mn-MN", + # Nepali (Preview) + Language.NE: "ne-NP", + Language.NE_NP: "ne-NP", + # Norwegian + Language.NO: "nb-NO", # Preview: Bokmål + Language.NB: "nb-NO", + Language.NB_NO: "nb-NO", + Language.NN: "nn-NO", # Preview: Nynorsk + Language.NN_NO: "nn-NO", + # Odia (Preview) + Language.OR: "or-IN", + Language.OR_IN: "or-IN", + # Pashto (Preview) + Language.PS: "ps-AF", + Language.PS_AF: "ps-AF", + # Persian (Preview) + Language.FA: "fa-IR", + Language.FA_IR: "fa-IR", + # Polish (GA) + Language.PL: "pl-PL", + Language.PL_PL: "pl-PL", + # Portuguese + Language.PT: "pt-BR", # GA: Brazil + Language.PT_BR: "pt-BR", + Language.PT_PT: "pt-PT", # Preview: Portugal + # Punjabi (Preview) + Language.PA: "pa-IN", + Language.PA_IN: "pa-IN", + # Romanian (GA) + Language.RO: "ro-RO", + Language.RO_RO: "ro-RO", + # Russian (GA) + Language.RU: "ru-RU", + Language.RU_RU: "ru-RU", + # Serbian (Preview) + Language.SR: "sr-RS", + Language.SR_RS: "sr-RS", + # Sindhi (Preview) + Language.SD: "sd-IN", + Language.SD_IN: "sd-IN", + # Sinhala (Preview) + Language.SI: "si-LK", + Language.SI_LK: "si-LK", + # Slovak (Preview) + Language.SK: "sk-SK", + Language.SK_SK: "sk-SK", + # Slovenian (Preview) + Language.SL: "sl-SI", + Language.SL_SI: "sl-SI", + # Spanish + Language.ES: "es-ES", # GA + Language.ES_ES: "es-ES", + Language.ES_419: "es-419", # Preview: Latin America + Language.ES_MX: "es-MX", # Preview: Mexico + # Swahili (Preview) + Language.SW: "sw-KE", + Language.SW_KE: "sw-KE", + # Swedish (Preview) + Language.SV: "sv-SE", + Language.SV_SE: "sv-SE", + # Tamil (GA) + Language.TA: "ta-IN", + Language.TA_IN: "ta-IN", + # Telugu (GA) + Language.TE: "te-IN", + Language.TE_IN: "te-IN", + # Thai (GA) + Language.TH: "th-TH", + Language.TH_TH: "th-TH", + # Turkish (GA) + Language.TR: "tr-TR", + Language.TR_TR: "tr-TR", + # Ukrainian (GA) + Language.UK: "uk-UA", + Language.UK_UA: "uk-UA", + # Urdu (Preview) + Language.UR: "ur-PK", + Language.UR_PK: "ur-PK", + # Vietnamese (GA) + Language.VI: "vi-VN", + Language.VI_VN: "vi-VN", + } + + return resolve_language(language, LANGUAGE_MAP, use_base_code=False) + + class GoogleHttpTTSService(TTSService): """Google Cloud Text-to-Speech HTTP service with SSML support. @@ -498,7 +742,139 @@ class GoogleHttpTTSService(TTSService): yield ErrorFrame(error=error_message) -class GoogleTTSService(TTSService): +class GoogleBaseTTSService(TTSService): + """Base class for Google Cloud Text-to-Speech streaming services. + + Provides shared streaming synthesis logic for Google TTS services. + This is an abstract base class. Use GoogleTTSService or GeminiTTSService instead. + """ + + def _create_client( + self, credentials: Optional[str], credentials_path: Optional[str] + ) -> texttospeech_v1.TextToSpeechAsyncClient: + """Create authenticated Google Text-to-Speech client. + + Args: + credentials: JSON string with service account credentials. + credentials_path: Path to service account JSON file. + + Returns: + Authenticated TextToSpeechAsyncClient instance. + + Raises: + ValueError: If no valid credentials are provided. + """ + creds: Optional[service_account.Credentials] = None + + if credentials: + # Use provided credentials JSON string + json_account_info = json.loads(credentials) + creds = service_account.Credentials.from_service_account_info(json_account_info) + elif credentials_path: + # Use service account JSON file if provided + creds = service_account.Credentials.from_service_account_file(credentials_path) + else: + try: + creds, project_id = default( + scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + except GoogleAuthError: + pass + + if not creds: + raise ValueError("No valid credentials provided.") + + return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds) + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Google streaming TTS services support metrics generation. + """ + return True + + @property + def includes_inter_frame_spaces(self) -> bool: + """Indicates that Google and Gemini TTSTextFrames include necessary inter-frame spaces. + + Returns: + True, indicating that Google's text frames include necessary inter-frame spaces. + """ + return True + + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Google TTS language format. + + Args: + language: The language to convert. + + Returns: + The Google TTS-specific language code, or None if not supported. + """ + return language_to_google_tts_language(language) + + async def _stream_tts( + self, + streaming_config: texttospeech_v1.StreamingSynthesizeConfig, + text: str, + prompt: Optional[str] = None, + ) -> AsyncGenerator[Frame, None]: + """Shared streaming synthesis logic. + + Args: + streaming_config: The streaming configuration. + text: The text to synthesize. + prompt: Optional prompt for style instructions (Gemini only). + + Yields: + Frame: Audio frames containing the synthesized speech. + """ + config_request = texttospeech_v1.StreamingSynthesizeRequest( + streaming_config=streaming_config + ) + + async def request_generator(): + yield config_request + synthesis_input_params = {"text": text} + if prompt is not None: + synthesis_input_params["prompt"] = prompt + yield texttospeech_v1.StreamingSynthesizeRequest( + input=texttospeech_v1.StreamingSynthesisInput(**synthesis_input_params) + ) + + streaming_responses = await self._client.streaming_synthesize(request_generator()) + await self.start_tts_usage_metrics(text) + + yield TTSStartedFrame() + + audio_buffer = b"" + first_chunk_for_ttfb = False + + CHUNK_SIZE = self.chunk_size + + async for response in streaming_responses: + chunk = response.audio_content + if not chunk: + continue + + if not first_chunk_for_ttfb: + await self.stop_ttfb_metrics() + first_chunk_for_ttfb = True + + audio_buffer += chunk + while len(audio_buffer) >= CHUNK_SIZE: + piece = audio_buffer[:CHUNK_SIZE] + audio_buffer = audio_buffer[CHUNK_SIZE:] + yield TTSAudioRawFrame(piece, self.sample_rate, 1) + + if audio_buffer: + yield TTSAudioRawFrame(audio_buffer, self.sample_rate, 1) + + yield TTSStoppedFrame() + + +class GoogleTTSService(GoogleBaseTTSService): """Google Cloud Text-to-Speech streaming service. Provides real-time text-to-speech synthesis using Google Cloud's streaming API @@ -570,62 +946,6 @@ class GoogleTTSService(TTSService): credentials, credentials_path ) - def _create_client( - self, credentials: Optional[str], credentials_path: Optional[str] - ) -> texttospeech_v1.TextToSpeechAsyncClient: - creds: Optional[service_account.Credentials] = None - - # Create a Google Cloud service account for the Cloud Text-to-Speech API - # Using either the provided credentials JSON string or the path to a service account JSON - # file, create a Google Cloud service account and use it to authenticate with the API. - if credentials: - # Use provided credentials JSON string - json_account_info = json.loads(credentials) - creds = service_account.Credentials.from_service_account_info(json_account_info) - elif credentials_path: - # Use service account JSON file if provided - creds = service_account.Credentials.from_service_account_file(credentials_path) - else: - try: - creds, project_id = default( - scopes=["https://www.googleapis.com/auth/cloud-platform"] - ) - except GoogleAuthError: - pass - - if not creds: - raise ValueError("No valid credentials provided.") - - return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds) - - def can_generate_metrics(self) -> bool: - """Check if this service can generate processing metrics. - - Returns: - True, as Google streaming TTS service supports metrics generation. - """ - return True - - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Google TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Google's text frames include necessary inter-frame spaces. - """ - return True - - def language_to_service_language(self, language: Language) -> Optional[str]: - """Convert a Language enum to Google TTS language format. - - Args: - language: The language to convert. - - Returns: - The Google TTS-specific language code, or None if not supported. - """ - return language_to_google_tts_language(language) - async def _update_settings(self, settings: Mapping[str, Any]): """Override to handle speaking_rate updates for streaming API. @@ -657,6 +977,7 @@ class GoogleTTSService(TTSService): try: await self.start_ttfb_metrics() + # Build voice selection params if self._voice_cloning_key: voice_clone_params = texttospeech_v1.VoiceCloneParams( voice_cloning_key=self._voice_cloning_key @@ -669,6 +990,7 @@ class GoogleTTSService(TTSService): language_code=self._settings["language"], name=self._voice_id ) + # Create streaming config streaming_config = texttospeech_v1.StreamingSynthesizeConfig( voice=voice, streaming_audio_config=texttospeech_v1.StreamingAudioConfig( @@ -677,45 +999,10 @@ class GoogleTTSService(TTSService): speaking_rate=self._settings["speaking_rate"], ), ) - config_request = texttospeech_v1.StreamingSynthesizeRequest( - streaming_config=streaming_config - ) - async def request_generator(): - yield config_request - yield texttospeech_v1.StreamingSynthesizeRequest( - input=texttospeech_v1.StreamingSynthesisInput(text=text) - ) - - streaming_responses = await self._client.streaming_synthesize(request_generator()) - await self.start_tts_usage_metrics(text) - - yield TTSStartedFrame() - - audio_buffer = b"" - first_chunk_for_ttfb = False - - CHUNK_SIZE = self.chunk_size - - async for response in streaming_responses: - chunk = response.audio_content - if not chunk: - continue - - if not first_chunk_for_ttfb: - await self.stop_ttfb_metrics() - first_chunk_for_ttfb = True - - audio_buffer += chunk - while len(audio_buffer) >= CHUNK_SIZE: - piece = audio_buffer[:CHUNK_SIZE] - audio_buffer = audio_buffer[CHUNK_SIZE:] - yield TTSAudioRawFrame(piece, self.sample_rate, 1) - - if audio_buffer: - yield TTSAudioRawFrame(audio_buffer, self.sample_rate, 1) - - yield TTSStoppedFrame() + # Use base class streaming logic + async for frame in self._stream_tts(streaming_config, text): + yield frame except Exception as e: logger.exception(f"{self} error generating TTS: {e}") @@ -723,25 +1010,29 @@ class GoogleTTSService(TTSService): yield ErrorFrame(error=error_message) -class GeminiTTSService(TTSService): - """Gemini Text-to-Speech service using Gemini TTS models. +class GeminiTTSService(GoogleBaseTTSService): + """Gemini Text-to-Speech streaming service using Gemini TTS models. - Provides text-to-speech synthesis using Gemini's TTS-specific models - (gemini-2.5-flash-preview-tts and gemini-2.5-pro-preview-tts) with - support for natural voice control, multiple speakers, and voice styles. + Provides real-time text-to-speech synthesis using Gemini's TTS-specific models + (gemini-2.5-flash-tts and gemini-2.5-pro-tts) with support for natural + voice control, prompts for style instructions, expressive markup tags, + and multi-speaker conversations. Note: - Requires Google AI API key. This uses the Gemini API, not Google Cloud TTS. - Audio-out is currently a preview feature. + Requires Google Cloud credentials via service account JSON, credentials file, + or default application credentials (GOOGLE_APPLICATION_CREDENTIALS). + + Uses the Google Cloud Text-to-Speech streaming API for low-latency synthesis. Example:: tts = GeminiTTSService( - api_key="your-google-ai-api-key", - model="gemini-2.5-flash-preview-tts", + credentials_path="/path/to/service-account.json", + model="gemini-2.5-flash-tts", voice_id="Kore", params=GeminiTTSService.InputParams( language=Language.EN_US, + prompt="Say this in a friendly and helpful tone" ) ) """ @@ -750,36 +1041,36 @@ class GeminiTTSService(TTSService): # List of available Gemini TTS voices AVAILABLE_VOICES = [ - "Zephyr", - "Puck", + "Achernar", + "Achird", + "Algenib", + "Algieba", + "Alnilam", + "Aoede", + "Autonoe", + "Callirhoe", "Charon", - "Kore", + "Despina", + "Enceladus", + "Erinome", "Fenrir", + "Gacrux", + "Iapetus", + "Kore", + "Laomedeia", "Leda", "Orus", - "Aoede", - "Callirhoe", - "Autonoe", - "Enceladus", - "Iapetus", - "Umbriel", - "Algieba", - "Despina", - "Erinome", - "Algenib", - "Rasalgethi", - "Laomedeia", - "Achernar", - "Alnilam", - "Schedar", - "Gacrux", + "Puck", "Pulcherrima", - "Achird", - "Zubenelgenubi", - "Vindemiatrix", + "Rasalgethi", "Sadachbia", "Sadaltager", + "Schedar", "Sulafar", + "Umbriel", + "Vindemiatrix", + "Zephyr", + "Zubenelgenubi", ] class InputParams(BaseModel): @@ -787,19 +1078,23 @@ class GeminiTTSService(TTSService): Parameters: language: Language for synthesis. Defaults to English. + prompt: Optional style instructions for how to synthesize the content. multi_speaker: Whether to enable multi-speaker support. speaker_configs: List of speaker configurations for multi-speaker mode. """ language: Optional[Language] = Language.EN + prompt: Optional[str] = None multi_speaker: bool = False speaker_configs: Optional[List[dict]] = None def __init__( self, *, - api_key: str, - model: str = "gemini-2.5-flash-preview-tts", + api_key: Optional[str] = None, + model: str = "gemini-2.5-flash-tts", + credentials: Optional[str] = None, + credentials_path: Optional[str] = None, voice_id: str = "Kore", sample_rate: Optional[int] = None, params: Optional[InputParams] = None, @@ -808,14 +1103,30 @@ class GeminiTTSService(TTSService): """Initializes the Gemini TTS service. Args: - api_key: Google AI API key for authentication. + api_key: + + .. deprecated:: 0.0.95 + The `api_key` parameter is deprecated. Use `credentials` or + `credentials_path` instead for Google Cloud authentication. + model: Gemini TTS model to use. Must be a TTS model like - "gemini-2.5-flash-preview-tts" or "gemini-2.5-pro-preview-tts". + "gemini-2.5-flash-tts" or "gemini-2.5-pro-tts". + credentials: JSON string containing Google Cloud service account credentials. + credentials_path: Path to Google Cloud service account JSON file. voice_id: Voice name from the available Gemini voices. sample_rate: Audio sample rate in Hz. If None, uses Google's default 24kHz. params: TTS configuration parameters. **kwargs: Additional arguments passed to parent TTSService. """ + # Handle deprecated api_key parameter + if api_key is not None: + warnings.warn( + "The 'api_key' parameter is deprecated and will be removed in a future version. " + "Use 'credentials' or 'credentials_path' instead for Google Cloud authentication.", + DeprecationWarning, + stacklevel=2, + ) + if sample_rate and sample_rate != self.GOOGLE_SAMPLE_RATE: logger.warning( f"Google TTS only supports {self.GOOGLE_SAMPLE_RATE}Hz sample rate. " @@ -828,35 +1139,20 @@ class GeminiTTSService(TTSService): if voice_id not in self.AVAILABLE_VOICES: logger.warning(f"Voice '{voice_id}' not in known voices list. Using anyway.") - self._api_key = api_key self._model = model self._voice_id = voice_id self._settings = { "language": self.language_to_service_language(params.language) if params.language else "en-US", + "prompt": params.prompt, "multi_speaker": params.multi_speaker, "speaker_configs": params.speaker_configs, } - self._client = genai.Client(api_key=api_key) - - def can_generate_metrics(self) -> bool: - """Check if this service can generate processing metrics. - - Returns: - True, as Gemini TTS service supports metrics generation. - """ - return True - - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Gemini TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Gemini's text frames include necessary inter-frame spaces. - """ - return True + self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( + credentials, credentials_path + ) def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Gemini TTS language format. @@ -867,7 +1163,7 @@ class GeminiTTSService(TTSService): Returns: The Gemini TTS-specific language code, or None if not supported. """ - return language_to_google_tts_language(language) + return language_to_gemini_tts_language(language) def set_voice(self, voice_id: str): """Set the voice for TTS generation. @@ -892,88 +1188,73 @@ class GeminiTTSService(TTSService): f"Current rate of {self.sample_rate}Hz may cause issues." ) - @traced_tts - async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate speech from text using Gemini TTS models. + async def _update_settings(self, settings: Mapping[str, Any]): + """Override to handle prompt updates. Args: - text: The text to synthesize into speech. Can include natural language - instructions for style, tone, etc. + settings: Dictionary of settings to update. Can include 'prompt' (str) + """ + if "prompt" in settings: + self._settings["prompt"] = settings["prompt"] + await super()._update_settings(settings) + + @traced_tts + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate streaming speech from text using Gemini TTS models. + + Args: + text: The text to synthesize into speech. Can include markup tags + like [sigh], [laughing], [whispering] for expressive control. Yields: - Frame: Audio frames containing the synthesized speech. + Frame: Audio frames containing the synthesized speech as it's generated. """ logger.debug(f"{self}: Generating TTS [{text}]") try: await self.start_ttfb_metrics() - # Build the speech config + # Build voice selection params if self._settings["multi_speaker"] and self._settings["speaker_configs"]: # Multi-speaker mode speaker_voice_configs = [] for speaker_config in self._settings["speaker_configs"]: speaker_voice_configs.append( - types.SpeakerVoiceConfig( - speaker=speaker_config["speaker"], - voice_config=types.VoiceConfig( - prebuilt_voice_config=types.PrebuiltVoiceConfig( - voice_name=speaker_config.get("voice_id", self._voice_id) - ) - ), + texttospeech_v1.MultispeakerPrebuiltVoice( + speaker_alias=speaker_config["speaker_alias"], + speaker_id=speaker_config.get("speaker_id", self._voice_id), ) ) - speech_config = types.SpeechConfig( - multi_speaker_voice_config=types.MultiSpeakerVoiceConfig( - speaker_voice_configs=speaker_voice_configs - ) + multi_speaker_voice_config = texttospeech_v1.MultiSpeakerVoiceConfig( + speaker_voice_configs=speaker_voice_configs + ) + + voice = texttospeech_v1.VoiceSelectionParams( + language_code=self._settings["language"], + model_name=self._model, + multi_speaker_voice_config=multi_speaker_voice_config, ) else: # Single speaker mode - speech_config = types.SpeechConfig( - voice_config=types.VoiceConfig( - prebuilt_voice_config=types.PrebuiltVoiceConfig(voice_name=self._voice_id) - ) + voice = texttospeech_v1.VoiceSelectionParams( + language_code=self._settings["language"], + name=self._voice_id, + model_name=self._model, ) - # Create the generation config - generation_config = types.GenerateContentConfig( - response_modalities=["AUDIO"], - speech_config=speech_config, + # Create streaming config + streaming_config = texttospeech_v1.StreamingSynthesizeConfig( + voice=voice, + streaming_audio_config=texttospeech_v1.StreamingAudioConfig( + audio_encoding=texttospeech_v1.AudioEncoding.PCM, + sample_rate_hertz=self.sample_rate, + ), ) - # Generate the content - response = await self._client.aio.models.generate_content( - model=self._model, - contents=text, - config=generation_config, - ) - - await self.start_tts_usage_metrics(text) - - yield TTSStartedFrame() - - # Extract audio data from response - if response.candidates and len(response.candidates) > 0: - candidate = response.candidates[0] - if candidate.content and candidate.content.parts: - for part in candidate.content.parts: - if part.inline_data and part.inline_data.mime_type.startswith("audio/"): - audio_data = part.inline_data.data - await self.stop_ttfb_metrics() - - # Gemini TTS returns PCM audio data, chunk it appropriately - CHUNK_SIZE = self.chunk_size - - for i in range(0, len(audio_data), CHUNK_SIZE): - chunk = audio_data[i : i + CHUNK_SIZE] - if not chunk: - break - frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) - yield frame - - yield TTSStoppedFrame() + # Use base class streaming logic with prompt support + async for frame in self._stream_tts(streaming_config, text, self._settings["prompt"]): + yield frame except Exception as e: logger.exception(f"{self} error generating TTS: {e}") diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py index cc84346c7..01c75d49f 100644 --- a/src/pipecat/transcriptions/language.py +++ b/src/pipecat/transcriptions/language.py @@ -66,6 +66,7 @@ class Language(StrEnum): AR_TN = "ar-TN" AR_XA = "ar-XA" AR_YE = "ar-YE" + AR_001 = "ar-001" # Assamese AS = "as" @@ -83,6 +84,7 @@ class Language(StrEnum): # Belarusian BE = "be" + BE_BY = "be-BY" # Bulgarian BG = "bg" @@ -109,6 +111,7 @@ class Language(StrEnum): # Cebuano CEB = "ceb" + CEB_PH = "ceb-PH" # Mandarin Chinese CMN = "cmn" @@ -181,6 +184,7 @@ class Language(StrEnum): ES_US = "es-US" ES_UY = "es-UY" ES_VE = "es-VE" + ES_419 = "es-419" # Estonian ET = "et" @@ -250,6 +254,7 @@ class Language(StrEnum): # Haitian Creole HT = "ht" + HT_HT = "ht-HT" # Hungarian HU = "hu" @@ -288,6 +293,7 @@ class Language(StrEnum): # Javanese JV = "jv" JV_ID = "jv-ID" + JV_JV = "jv-JV" JW = "jw" # Fal requires for Javanese # Georgian @@ -309,6 +315,10 @@ class Language(StrEnum): KN = "kn" KN_IN = "kn-IN" + # Konkani + KOK = "kok" + KOK_IN = "kok-IN" + # Korean KO = "ko" KO_KR = "ko-KR" @@ -322,9 +332,11 @@ class Language(StrEnum): # Latin LA = "la" + LA_VA = "la-VA" # Luxembourgish LB = "lb" + LB_LU = "lb-LU" # Lingala LN = "ln" @@ -349,6 +361,7 @@ class Language(StrEnum): # Malagasy MG = "mg" + MG_MG = "mg-MG" # Maori MI = "mi" @@ -357,6 +370,10 @@ class Language(StrEnum): MK = "mk" MK_MK = "mk-MK" + # Maithili + MAI = "mai" + MAI_IN = "mai-IN" + # Malayalam ML = "ml" ML_IN = "ml-IN" @@ -387,6 +404,7 @@ class Language(StrEnum): NB_NO = "nb-NO" NO = "no" NN = "nn" # Norwegian Nynorsk + NN_NO = "nn-NO" # Nepali NE = "ne" @@ -440,6 +458,7 @@ class Language(StrEnum): # Sindhi SD = "sd" + SD_IN = "sd-IN" # Sinhala SI = "si" From fbbad27d3769a95d460ebe7a40c559aa33ce1a12 Mon Sep 17 00:00:00 2001 From: Corvin Jaedicke Date: Fri, 14 Nov 2025 13:30:06 +0100 Subject: [PATCH 032/110] add changelog info --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cca64597..9fb339a13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. +### Added + +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no + ONNX dependency or added processing complexity. + ## [0.0.94] - 2025-11-10 ### Deprecated From d01876ee6050d469fa72b10a2f312de337be4769 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 14 Nov 2025 12:13:49 -0500 Subject: [PATCH 033/110] Remove fallbacks in traced_llm --- src/pipecat/utils/tracing/service_decorators.py | 16 +++------------- uv.lock | 6 +++--- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/pipecat/utils/tracing/service_decorators.py b/src/pipecat/utils/tracing/service_decorators.py index 3c743c1a6..ead4aa9e8 100644 --- a/src/pipecat/utils/tracing/service_decorators.py +++ b/src/pipecat/utils/tracing/service_decorators.py @@ -23,7 +23,7 @@ if TYPE_CHECKING: from opentelemetry import context as context_api from opentelemetry import trace -from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_context import NOT_GIVEN, LLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.utils.tracing.service_attributes import ( add_gemini_live_span_attributes, @@ -399,11 +399,6 @@ def traced_llm(func: Optional[Callable] = None, *, name: Optional[str] = None) - if hasattr(self, "get_llm_adapter"): adapter = self.get_llm_adapter() messages = adapter.get_messages_for_logging(context) - elif hasattr(context, "get_messages"): - # Fallback for unknown context types - messages = context.get_messages() - elif hasattr(context, "messages"): - messages = context.messages # Serialize messages if available if messages: @@ -424,15 +419,10 @@ def traced_llm(func: Optional[Callable] = None, *, name: Optional[str] = None) - if hasattr(self, "get_llm_adapter") and hasattr(context, "tools"): adapter = self.get_llm_adapter() tools = adapter.from_standard_tools(context.tools) - elif hasattr(context, "tools"): - # Fallback for unknown context types - tools = context.tools # Serialize and count tools if available - # Check if tools is not None and not NOT_GIVEN (using attribute check as fallback) - if tools is not None and not ( - hasattr(tools, "__name__") and tools.__name__ == "NOT_GIVEN" - ): + # Check if tools is not None and not NOT_GIVEN + if tools is not None and tools is not NOT_GIVEN: serialized_tools = json.dumps(tools) tool_count = len(tools) if isinstance(tools, list) else 1 diff --git a/uv.lock b/uv.lock index 0dc6c74f8..eabe03714 100644 --- a/uv.lock +++ b/uv.lock @@ -36,12 +36,12 @@ wheels = [ [[package]] name = "aic-sdk" -version = "1.0.2" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/90/b02e853e863c303f8456c689b42ac24ad403b781adc9642d0a91ed4bed7e/aic_sdk-1.0.2.tar.gz", hash = "sha256:239097dd3aaa8a8a0fd7542b75d2510cb34144caec796370639b7c636acbc56e", size = 32059, upload-time = "2025-08-24T09:20:03.9Z" } +sdist = { url = "https://files.pythonhosted.org/packages/99/83/bf38b95d98c67b8ebc574fb4a4f23c07a3740b51992d7522976173d30b98/aic_sdk-1.1.0.tar.gz", hash = "sha256:04e08df695581c8cb4db8acca20e73815e9f449e7bd08e0162fd55518c727963", size = 34954, upload-time = "2025-11-11T20:45:24.25Z" } [[package]] name = "aioboto3" @@ -4647,7 +4647,7 @@ docs = [ [package.metadata] requires-dist = [ { name = "accelerate", marker = "extra == 'moondream'", specifier = "~=1.10.0" }, - { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.0.1" }, + { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.1.0" }, { name = "aioboto3", marker = "extra == 'aws'", specifier = "~=15.0.0" }, { name = "aiofiles", specifier = ">=24.1.0,<25" }, { name = "aiohttp", specifier = ">=3.11.12,<4" }, From d1116d149e62039cdbed116aec12deadbabf8794 Mon Sep 17 00:00:00 2001 From: Angad Singh Date: Fri, 14 Nov 2025 23:33:05 +0530 Subject: [PATCH 034/110] feat: Add ErrorFrame emission to TTS/STT services for pipeline error detection (#2881) * feat: Add ErrorFrame emission to TTS/STT services for pipeline error detection - Add ErrorFrame emission to all major TTS/STT services during initialization and runtime failures - Services updated: Cartesia, ElevenLabs, Deepgram, AssemblyAI, Rime, Azure - ErrorFrame objects emitted with fatal=False for graceful degradation - Enables on_pipeline_error event handler to detect service failures programmatically - Add comprehensive pytest test suite to verify ErrorFrame emission - Fixes issue where services failed gracefully but didn't emit ErrorFrame objects This allows developers to implement real-time error monitoring and alerting using the on_pipeline_error event handler introduced in v0.0.90. * Update STT and TTS services to use consistent error handling pattern - Improves error handling consistency across all services * Add changelog entry for STT/TTS error handling improvements * Linting issues Resolved * Azure STT ErrorFrames added with consistent patterns * Cartesia STT and Deepgram STT; additional fixes made * Removed Fatal Flags across services, removed duplication * Moving the changelog entry to the correct place. * Refactoring some classes to use yield instead of push_error directly. * Fixing ruff format. --------- Co-authored-by: Filipi Fuchter --- CHANGELOG.md | 3 ++ src/pipecat/services/assemblyai/stt.py | 19 ++++++--- src/pipecat/services/asyncai/tts.py | 16 +++++--- src/pipecat/services/aws/stt.py | 28 ++++++------- src/pipecat/services/azure/stt.py | 39 ++++++++++++------- src/pipecat/services/azure/tts.py | 7 +++- src/pipecat/services/cartesia/stt.py | 9 ++++- src/pipecat/services/cartesia/tts.py | 16 +++++--- src/pipecat/services/deepgram/flux/stt.py | 16 +++++--- src/pipecat/services/deepgram/stt.py | 2 +- src/pipecat/services/deepgram/tts.py | 4 +- src/pipecat/services/elevenlabs/stt.py | 4 +- src/pipecat/services/elevenlabs/tts.py | 23 +++++++---- src/pipecat/services/fal/stt.py | 4 +- src/pipecat/services/fish/tts.py | 16 +++++--- src/pipecat/services/gladia/stt.py | 10 +++-- .../services/google/gemini_live/llm.py | 6 +-- src/pipecat/services/google/stt.py | 13 ++++--- src/pipecat/services/google/tts.py | 6 +-- src/pipecat/services/groq/tts.py | 9 ++++- src/pipecat/services/hume/tts.py | 4 +- src/pipecat/services/inworld/tts.py | 6 +-- src/pipecat/services/lmnt/tts.py | 12 ++++-- src/pipecat/services/minimax/tts.py | 4 +- src/pipecat/services/neuphonic/tts.py | 17 +++++--- src/pipecat/services/openai/realtime/llm.py | 8 ++-- src/pipecat/services/openai/tts.py | 3 +- .../services/openai_realtime_beta/openai.py | 8 ++-- src/pipecat/services/piper/tts.py | 6 +-- src/pipecat/services/playht/tts.py | 18 +++++---- src/pipecat/services/rime/tts.py | 16 +++++--- src/pipecat/services/riva/stt.py | 4 +- src/pipecat/services/riva/tts.py | 2 + src/pipecat/services/sarvam/tts.py | 23 ++++++----- src/pipecat/services/soniox/stt.py | 10 ++--- src/pipecat/services/speechmatics/stt.py | 12 ++++-- src/pipecat/services/ultravox/stt.py | 13 ++++--- src/pipecat/services/websocket_service.py | 2 +- src/pipecat/services/whisper/base_stt.py | 4 +- src/pipecat/services/whisper/stt.py | 4 +- src/pipecat/services/xtts/tts.py | 4 +- 41 files changed, 260 insertions(+), 170 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3748892b6..7cda253c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Updated all STT and TTS services to use consistent error handling pattern with + `push_error()` method for better pipeline error event integration. + - Added Hindi support for Rime TTS services. - Updated `GeminiTTSService` to use Google Cloud Text-to-Speech streaming API diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index b3f20800c..d78a42841 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -21,6 +21,7 @@ from pipecat import __version__ as pipecat_version from pipecat.frames.frames import ( CancelFrame, EndFrame, + ErrorFrame, Frame, InterimTranscriptionFrame, StartFrame, @@ -205,8 +206,9 @@ class AssemblyAISTTService(STTService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"Failed to connect to AssemblyAI: {e}") + logger.error(f"{self} exception: {e}") self._connected = False + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) raise async def _disconnect(self): @@ -231,7 +233,8 @@ class AssemblyAISTTService(STTService): logger.warning("Timed out waiting for termination message from server") except Exception as e: - logger.warning(f"Error during termination handshake: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) if self._receive_task: await self.cancel_task(self._receive_task) @@ -239,7 +242,8 @@ class AssemblyAISTTService(STTService): await self._websocket.close() except Exception as e: - logger.error(f"Error during disconnect: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._websocket = None @@ -258,11 +262,13 @@ class AssemblyAISTTService(STTService): except websockets.exceptions.ConnectionClosedOK: break except Exception as e: - logger.error(f"Error processing WebSocket message: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) break except Exception as e: - logger.error(f"Fatal error in receive handler: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) def _parse_message(self, message: Dict[str, Any]) -> BaseMessage: """Parse a raw message into the appropriate message type.""" @@ -291,7 +297,8 @@ class AssemblyAISTTService(STTService): elif isinstance(parsed_message, TerminationMessage): await self._handle_termination(parsed_message) except Exception as e: - logger.error(f"Error handling message: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) async def _handle_termination(self, message: TerminationMessage): """Handle termination message.""" diff --git a/src/pipecat/services/asyncai/tts.py b/src/pipecat/services/asyncai/tts.py index 78cdd7ef8..8df597c56 100644 --- a/src/pipecat/services/asyncai/tts.py +++ b/src/pipecat/services/asyncai/tts.py @@ -237,7 +237,8 @@ class AsyncAITTSService(InterruptibleTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} initialization error: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -249,7 +250,8 @@ class AsyncAITTSService(InterruptibleTTSService): logger.debug("Disconnecting from Async") await self._websocket.close() except Exception as e: - logger.error(f"{self} error closing websocket: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._websocket = None self._started = False @@ -297,7 +299,7 @@ class AsyncAITTSService(InterruptibleTTSService): logger.error(f"{self} error: {msg}") await self.push_frame(TTSStoppedFrame()) await self.stop_all_metrics() - await self.push_error(ErrorFrame(f"{self} error: {msg['message']}")) + await self.push_error(ErrorFrame(error=f"{self} error: {msg['message']}")) else: logger.error(f"{self} error, unknown message type: {msg}") @@ -342,7 +344,8 @@ class AsyncAITTSService(InterruptibleTTSService): await self._get_websocket().send(msg) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} error sending message: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() @@ -350,6 +353,7 @@ class AsyncAITTSService(InterruptibleTTSService): yield None except Exception as e: logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") class AsyncAIHttpTTSService(TTSService): @@ -492,7 +496,7 @@ class AsyncAIHttpTTSService(TTSService): if response.status != 200: error_text = await response.text() logger.error(f"Async API error: {error_text}") - await self.push_error(ErrorFrame(f"Async API error: {error_text}")) + await self.push_error(ErrorFrame(error=f"Async API error: {error_text}")) raise Exception(f"Async API returned status {response.status}: {error_text}") audio_data = await response.read() @@ -509,7 +513,7 @@ class AsyncAIHttpTTSService(TTSService): except Exception as e: logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(f"Error generating TTS: {e}")) + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index b1e0b5ba7..8d89d3198 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -140,7 +140,8 @@ class AWSTranscribeSTTService(STTService): return logger.warning("WebSocket connection not established after connect") except Exception as e: - logger.error(f"Failed to connect (attempt {retry_count + 1}/{max_retries}): {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) retry_count += 1 if retry_count < max_retries: await asyncio.sleep(1) # Wait before retrying @@ -181,8 +182,8 @@ class AWSTranscribeSTTService(STTService): try: await self._connect() except Exception as e: - logger.error(f"Failed to reconnect: {e}") - yield ErrorFrame("Failed to reconnect to AWS Transcribe", fatal=False) + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") return # Format the audio data according to AWS event stream format @@ -199,13 +200,13 @@ class AWSTranscribeSTTService(STTService): await self._disconnect() # Don't yield error here - we'll retry on next frame except Exception as e: - logger.error(f"Error sending audio: {e}") - yield ErrorFrame(f"AWS Transcribe error: {str(e)}", fatal=False) + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") await self._disconnect() except Exception as e: - logger.error(f"Error in run_stt: {e}") - yield ErrorFrame(f"AWS Transcribe error: {str(e)}", fatal=False) + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") await self._disconnect() async def _connect(self): @@ -288,7 +289,8 @@ class AWSTranscribeSTTService(STTService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} Failed to connect to AWS Transcribe: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) await self._disconnect() raise @@ -308,7 +310,8 @@ class AWSTranscribeSTTService(STTService): await self._ws_client.send(json.dumps(end_stream)) await self._ws_client.close() except Exception as e: - logger.warning(f"{self} Error closing WebSocket connection: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._ws_client = None await self._call_event_handler("on_disconnected") @@ -527,9 +530,7 @@ class AWSTranscribeSTTService(STTService): elif headers.get(":message-type") == "exception": error_msg = payload.get("Message", "Unknown error") logger.error(f"{self} Exception from AWS: {error_msg}") - await self.push_frame( - ErrorFrame(f"AWS Transcribe error: {error_msg}", fatal=False) - ) + await self.push_frame(ErrorFrame(f"AWS Transcribe error: {error_msg}")) else: logger.debug(f"{self} Other message type received: {headers}") logger.debug(f"{self} Payload: {payload}") @@ -537,5 +538,6 @@ class AWSTranscribeSTTService(STTService): logger.error(f"{self} WebSocket connection closed in receive loop: {e}") break except Exception as e: - logger.error(f"{self} Unexpected error in receive loop: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) break diff --git a/src/pipecat/services/azure/stt.py b/src/pipecat/services/azure/stt.py index 586a94e44..85a0508a0 100644 --- a/src/pipecat/services/azure/stt.py +++ b/src/pipecat/services/azure/stt.py @@ -18,6 +18,7 @@ from loguru import logger from pipecat.frames.frames import ( CancelFrame, EndFrame, + ErrorFrame, Frame, InterimTranscriptionFrame, StartFrame, @@ -111,13 +112,17 @@ class AzureSTTService(STTService): audio: Raw audio bytes to process. Yields: - None - actual transcription frames are pushed via callbacks. + Frame: Either None for successful processing or ErrorFrame on failure. """ - await self.start_processing_metrics() - await self.start_ttfb_metrics() - if self._audio_stream: - self._audio_stream.write(audio) - yield None + try: + await self.start_processing_metrics() + await self.start_ttfb_metrics() + if self._audio_stream: + self._audio_stream.write(audio) + yield None + except Exception as e: + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") async def start(self, frame: StartFrame): """Start the speech recognition service. @@ -133,17 +138,21 @@ class AzureSTTService(STTService): if self._audio_stream: return - stream_format = AudioStreamFormat(samples_per_second=self.sample_rate, channels=1) - self._audio_stream = PushAudioInputStream(stream_format) + try: + stream_format = AudioStreamFormat(samples_per_second=self.sample_rate, channels=1) + self._audio_stream = PushAudioInputStream(stream_format) - audio_config = AudioConfig(stream=self._audio_stream) + audio_config = AudioConfig(stream=self._audio_stream) - self._speech_recognizer = SpeechRecognizer( - speech_config=self._speech_config, audio_config=audio_config - ) - self._speech_recognizer.recognizing.connect(self._on_handle_recognizing) - self._speech_recognizer.recognized.connect(self._on_handle_recognized) - self._speech_recognizer.start_continuous_recognition_async() + self._speech_recognizer = SpeechRecognizer( + speech_config=self._speech_config, audio_config=audio_config + ) + self._speech_recognizer.recognizing.connect(self._on_handle_recognizing) + self._speech_recognizer.recognized.connect(self._on_handle_recognized) + self._speech_recognizer.start_continuous_recognition_async() + except Exception as e: + logger.error(f"{self} exception during initialization: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) async def stop(self, frame: EndFrame): """Stop the speech recognition service. diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index d0ae42796..8ac8b70e3 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -337,7 +337,7 @@ class AzureTTSService(AzureBaseTTSService): if self._speech_synthesizer is None: error_msg = "Speech synthesizer not initialized." logger.error(error_msg) - yield ErrorFrame(error_msg) + yield ErrorFrame(error=error_msg) return try: @@ -364,13 +364,15 @@ class AzureTTSService(AzureBaseTTSService): yield TTSStoppedFrame() except Exception as e: - logger.error(f"{self} error during synthesis: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") yield TTSStoppedFrame() # Could add reconnection logic here if needed return except Exception as e: logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") class AzureHttpTTSService(AzureBaseTTSService): @@ -448,3 +450,4 @@ class AzureHttpTTSService(AzureBaseTTSService): logger.warning(f"Speech synthesis canceled: {cancellation_details.reason}") if cancellation_details.reason == CancellationReason.Error: logger.error(f"{self} error: {cancellation_details.error_details}") + yield ErrorFrame(error=f"{self} error: {cancellation_details.error_details}") diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index b4e232c4a..a2ae9432f 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -20,6 +20,7 @@ from loguru import logger from pipecat.frames.frames import ( CancelFrame, EndFrame, + ErrorFrame, Frame, InterimTranscriptionFrame, StartFrame, @@ -275,7 +276,8 @@ class CartesiaSTTService(WebsocketSTTService): self._websocket = await websocket_connect(ws_url, additional_headers=headers) await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self}: unable to connect to Cartesia: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) async def _disconnect_websocket(self): try: @@ -284,6 +286,7 @@ class CartesiaSTTService(WebsocketSTTService): await self._websocket.close() except Exception as e: logger.error(f"{self} error closing websocket: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._websocket = None await self._call_event_handler("on_disconnected") @@ -315,7 +318,9 @@ class CartesiaSTTService(WebsocketSTTService): await self._on_transcript(data) elif data["type"] == "error": - logger.error(f"Cartesia error: {data.get('message', 'Unknown error')}") + error_msg = data.get("message", "Unknown error") + logger.error(f"Cartesia error: {error_msg}") + await self.push_error(ErrorFrame(error=error_msg)) @traced_stt async def _handle_transcription( diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 9e7f6b37c..f8881200c 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -397,7 +397,8 @@ class CartesiaTTSService(AudioContextWordTTSService): ) await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} initialization error: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -409,7 +410,8 @@ class CartesiaTTSService(AudioContextWordTTSService): logger.debug("Disconnecting from Cartesia") await self._websocket.close() except Exception as e: - logger.error(f"{self} error closing websocket: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._context_id = None self._websocket = None @@ -465,7 +467,7 @@ class CartesiaTTSService(AudioContextWordTTSService): logger.error(f"{self} error: {msg}") await self.push_frame(TTSStoppedFrame()) await self.stop_all_metrics() - await self.push_error(ErrorFrame(f"{self} error: {msg['error']}")) + await self.push_error(ErrorFrame(error=f"{self} error: {msg['error']}")) self._context_id = None else: logger.error(f"{self} error, unknown message type: {msg}") @@ -506,7 +508,8 @@ class CartesiaTTSService(AudioContextWordTTSService): await self._get_websocket().send(msg) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} error sending message: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() @@ -514,6 +517,7 @@ class CartesiaTTSService(AudioContextWordTTSService): yield None except Exception as e: logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") class CartesiaHttpTTSService(TTSService): @@ -705,7 +709,7 @@ class CartesiaHttpTTSService(TTSService): if response.status != 200: error_text = await response.text() logger.error(f"Cartesia API error: {error_text}") - await self.push_error(ErrorFrame(f"Cartesia API error: {error_text}")) + await self.push_error(ErrorFrame(error=f"Cartesia API error: {error_text}")) raise Exception(f"Cartesia API returned status {response.status}: {error_text}") audio_data = await response.read() @@ -722,7 +726,7 @@ class CartesiaHttpTTSService(TTSService): except Exception as e: logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(f"Error generating TTS: {e}")) + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index 7adc0f35a..ac6e8bb08 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -191,7 +191,8 @@ class DeepgramFluxSTTService(WebsocketSTTService): await self._disconnect_websocket() except Exception as e: - logger.error(f"Error during disconnect: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: # Reset state only after everything is cleaned up self._websocket = None @@ -214,7 +215,8 @@ class DeepgramFluxSTTService(WebsocketSTTService): logger.debug("Connected to Deepgram Flux Websocket") await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} initialization error: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -233,6 +235,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): await self._websocket.close() except Exception as e: logger.error(f"{self} error closing websocket: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._websocket = None await self._call_event_handler("on_disconnected") @@ -333,14 +336,14 @@ class DeepgramFluxSTTService(WebsocketSTTService): """ if not self._websocket: logger.error("Not connected to Deepgram Flux.") - yield ErrorFrame("Not connected to Deepgram Flux.", fatal=True) + yield ErrorFrame("Not connected to Deepgram Flux.") return try: await self._websocket.send(audio) except Exception as e: - logger.error(f"Failed to send audio to Flux: {e}") - yield ErrorFrame(f"Failed to send audio to Flux: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") return yield None @@ -417,7 +420,8 @@ class DeepgramFluxSTTService(WebsocketSTTService): # Skip malformed messages continue except Exception as e: - logger.error(f"Error processing message: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) # Error will be handled inside WebsocketService->_receive_task_handler raise else: diff --git a/src/pipecat/services/deepgram/stt.py b/src/pipecat/services/deepgram/stt.py index 6b1ea2f21..0a4cd7e6e 100644 --- a/src/pipecat/services/deepgram/stt.py +++ b/src/pipecat/services/deepgram/stt.py @@ -256,7 +256,7 @@ class DeepgramSTTService(STTService): async def _on_error(self, *args, **kwargs): error: ErrorResponse = kwargs["error"] logger.warning(f"{self} connection error, will retry: {error}") - await self.push_error(ErrorFrame(f"{error}")) + await self.push_error(ErrorFrame(error=f"{error}")) await self.stop_all_metrics() # NOTE(aleix): we don't disconnect (i.e. call finish on the connection) # because this triggers more errors internally in the Deepgram SDK. So, diff --git a/src/pipecat/services/deepgram/tts.py b/src/pipecat/services/deepgram/tts.py index 2c816e4a9..f6045c1f3 100644 --- a/src/pipecat/services/deepgram/tts.py +++ b/src/pipecat/services/deepgram/tts.py @@ -125,8 +125,8 @@ class DeepgramTTSService(TTSService): yield TTSStoppedFrame() except Exception as e: - logger.exception(f"{self} exception: {e}") - yield ErrorFrame(f"Error getting audio: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") class DeepgramHttpTTSService(TTSService): diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py index 3b3349ba2..8cbb40d63 100644 --- a/src/pipecat/services/elevenlabs/stt.py +++ b/src/pipecat/services/elevenlabs/stt.py @@ -351,8 +351,8 @@ class ElevenLabsSTTService(SegmentedSTTService): ) except Exception as e: - logger.error(f"ElevenLabs STT error: {e}") - yield ErrorFrame(f"ElevenLabs STT error: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") def audio_format_from_sample_rate(sample_rate: int) -> str: diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 44017e264..bbe05f9dc 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -424,7 +424,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService): json.dumps({"context_id": self._context_id, "close_context": True}) ) except Exception as e: - logger.warning(f"Error closing context for voice settings update: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._context_id = None self._started = False @@ -535,8 +536,9 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} initialization error: {e}") + logger.error(f"{self} exception: {e}") self._websocket = None + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): @@ -551,7 +553,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self._websocket.close() logger.debug("Disconnected from ElevenLabs") except Exception as e: - logger.error(f"{self} error closing websocket: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._started = False self._context_id = None @@ -581,7 +584,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService): json.dumps({"context_id": self._context_id, "close_context": True}) ) except Exception as e: - logger.error(f"Error closing context on interruption: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._context_id = None self._started = False self._partial_word = "" @@ -736,13 +740,15 @@ class ElevenLabsTTSService(AudioContextWordTTSService): else: await self._send_text(text) except Exception as e: - logger.error(f"{self} error sending message: {e}") + logger.error(f"{self} exception: {e}") yield TTSStoppedFrame() + yield ErrorFrame(error=f"{self} error: {e}") self._started = False return yield None except Exception as e: logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") class ElevenLabsHttpTTSService(WordTTSService): @@ -1085,7 +1091,8 @@ class ElevenLabsHttpTTSService(WordTTSService): logger.warning(f"Failed to parse JSON from stream: {e}") continue except Exception as e: - logger.error(f"Error processing response: {e}", exc_info=True) + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") continue # After processing all chunks, emit any remaining partial word @@ -1109,8 +1116,8 @@ class ElevenLabsHttpTTSService(WordTTSService): self._previous_text = text except Exception as e: - logger.error(f"Error in run_tts: {e}") - yield ErrorFrame(error=str(e)) + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") finally: await self.stop_ttfb_metrics() # Let the parent class handle TTSStoppedFrame diff --git a/src/pipecat/services/fal/stt.py b/src/pipecat/services/fal/stt.py index f4a708e23..8b84aaeeb 100644 --- a/src/pipecat/services/fal/stt.py +++ b/src/pipecat/services/fal/stt.py @@ -290,5 +290,5 @@ class FalSTTService(SegmentedSTTService): ) except Exception as e: - logger.error(f"Fal Wizper error: {e}") - yield ErrorFrame(f"Fal Wizper error: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index 1abe6aca1..0acb12a96 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -237,7 +237,8 @@ class FishAudioTTSService(InterruptibleTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"Fish Audio initialization error: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -251,7 +252,8 @@ class FishAudioTTSService(InterruptibleTTSService): await self._websocket.send(ormsgpack.packb(stop_message)) await self._websocket.close() except Exception as e: - logger.error(f"Error closing websocket: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._request_id = None self._started = False @@ -293,7 +295,8 @@ class FishAudioTTSService(InterruptibleTTSService): continue except Exception as e: - logger.error(f"Error processing message: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: @@ -329,7 +332,8 @@ class FishAudioTTSService(InterruptibleTTSService): flush_message = {"event": "flush"} await self._get_websocket().send(ormsgpack.packb(flush_message)) except Exception as e: - logger.error(f"{self} error sending message: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() @@ -337,5 +341,5 @@ class FishAudioTTSService(InterruptibleTTSService): yield None except Exception as e: - logger.error(f"Error generating TTS: {e}") - yield ErrorFrame(f"Error: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") diff --git a/src/pipecat/services/gladia/stt.py b/src/pipecat/services/gladia/stt.py index cd59a3b74..624745293 100644 --- a/src/pipecat/services/gladia/stt.py +++ b/src/pipecat/services/gladia/stt.py @@ -23,6 +23,7 @@ from pipecat import __version__ as pipecat_version from pipecat.frames.frames import ( CancelFrame, EndFrame, + ErrorFrame, Frame, InterimTranscriptionFrame, StartFrame, @@ -467,7 +468,8 @@ class GladiaSTTService(STTService): break except Exception as e: - logger.error(f"Error in connection handler: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._connection_active = False if not self._should_reconnect: @@ -557,7 +559,8 @@ class GladiaSTTService(STTService): except websockets.exceptions.ConnectionClosed: logger.debug("Connection closed during keepalive") except Exception as e: - logger.error(f"Error in Gladia keepalive task: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) async def _receive_task_handler(self): try: @@ -620,7 +623,8 @@ class GladiaSTTService(STTService): # Expected when closing the connection pass except Exception as e: - logger.error(f"Error in Gladia WebSocket handler: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) async def _maybe_reconnect(self) -> bool: """Handle exponential backoff reconnection logic.""" diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 9c92076ab..11632968e 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -1174,7 +1174,7 @@ class GeminiLiveLLMService(LLMService): self._connection_task = self.create_task(self._connection_task_handler(config=config)) except Exception as e: - await self.push_error(ErrorFrame(error=f"{self} Initialization error: {e}", fatal=True)) + await self.push_error(ErrorFrame(error=f"{self} Initialization error: {e}")) async def _connection_task_handler(self, config: LiveConnectConfig): async with self._client.aio.live.connect(model=self._model_name, config=config) as session: @@ -1255,9 +1255,7 @@ class GeminiLiveLLMService(LLMService): f"Max consecutive failures ({MAX_CONSECUTIVE_FAILURES}) reached, " "treating as fatal error" ) - await self.push_error( - ErrorFrame(error=f"{self} Error in receive loop: {error}", fatal=True) - ) + await self.push_error(ErrorFrame(error=f"{self} Error in receive loop: {error}")) return False else: logger.info( diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index 59d1dcd63..beb5db740 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -774,7 +774,8 @@ class GoogleSTTService(STTService): yield cloud_speech.StreamingRecognizeRequest(audio=audio_data) except Exception as e: - logger.error(f"Error in request generator: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) raise async def _stream_audio(self): @@ -805,14 +806,15 @@ class GoogleSTTService(STTService): break except Exception as e: - logger.warning(f"{self} Reconnecting: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) await asyncio.sleep(1) # Brief delay before reconnecting self._stream_start_time = int(time.time() * 1000) except Exception as e: - logger.error(f"Error in streaming task: {e}") - await self.push_frame(ErrorFrame(str(e))) + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: """Process an audio chunk for STT transcription. @@ -900,7 +902,8 @@ class GoogleSTTService(STTService): ) raise except Exception as e: - logger.error(f"Error processing Google STT responses: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) # Re-raise the exception to let it propagate (e.g. in the case of a # timeout, propagate to _stream_audio to reconnect) raise diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index 4db084623..449b2bca2 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -746,7 +746,7 @@ class GoogleHttpTTSService(TTSService): yield TTSStoppedFrame() except Exception as e: - logger.exception(f"{self} error generating TTS: {e}") + logger.error(f"{self} exception: {e}") error_message = f"TTS generation error: {str(e)}" yield ErrorFrame(error=error_message) @@ -1014,7 +1014,7 @@ class GoogleTTSService(GoogleBaseTTSService): yield frame except Exception as e: - logger.exception(f"{self} error generating TTS: {e}") + logger.error(f"{self} exception: {e}") error_message = f"TTS generation error: {str(e)}" yield ErrorFrame(error=error_message) @@ -1266,6 +1266,6 @@ class GeminiTTSService(GoogleBaseTTSService): yield frame except Exception as e: - logger.exception(f"{self} error generating TTS: {e}") + logger.error(f"{self} exception: {e}") error_message = f"Gemini TTS generation error: {str(e)}" yield ErrorFrame(error=error_message) diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index 6bd49d1a3..d2dcb73a1 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -13,7 +13,13 @@ from typing import AsyncGenerator, Optional from loguru import logger from pydantic import BaseModel -from pipecat.frames.frames import Frame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame +from pipecat.frames.frames import ( + ErrorFrame, + Frame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, +) from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language from pipecat.utils.tracing.service_decorators import traced_tts @@ -150,5 +156,6 @@ class GroqTTSService(TTSService): yield TTSAudioRawFrame(bytes, frame_rate, channels) except Exception as e: logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") yield TTSStoppedFrame() diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index f8b2bbf27..6760c8121 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -225,8 +225,8 @@ class HumeTTSService(TTSService): self._audio_bytes = b"" except Exception as e: - logger.exception(f"{self} error generating TTS: {e}") - await self.push_error(ErrorFrame(f"Error generating TTS: {e}")) + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: # Ensure TTFB timer is stopped even on early failures await self.stop_ttfb_metrics() diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index 9bb939518..067ea6ab6 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -374,7 +374,7 @@ class InworldTTSService(TTSService): if response.status != 200: error_text = await response.text() logger.error(f"Inworld API error: {error_text}") - await self.push_error(ErrorFrame(f"Inworld API error: {error_text}")) + yield ErrorFrame(error=f"Inworld API error: {error_text}") return # ================================================================================ @@ -402,7 +402,7 @@ class InworldTTSService(TTSService): # ================================================================================ # Log any unexpected errors and notify the pipeline logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(f"Error generating TTS: {e}")) + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: # ================================================================================ # STEP 8: CLEANUP AND COMPLETION @@ -517,7 +517,7 @@ class InworldTTSService(TTSService): # Extract the base64-encoded audio content from response if "audioContent" not in response_data: logger.error("No audioContent in Inworld API response") - await self.push_error(ErrorFrame("No audioContent in response")) + await self.push_error(ErrorFrame(error="No audioContent in response")) return # ================================================================================ diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index 538c1ef93..d98d3f76c 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -223,7 +223,8 @@ class LmntTTSService(InterruptibleTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} initialization error: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -239,7 +240,8 @@ class LmntTTSService(InterruptibleTTSService): # await self._websocket.send(json.dumps({"eof": True})) await self._websocket.close() except Exception as e: - logger.error(f"{self} error closing websocket: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._started = False self._websocket = None @@ -276,7 +278,7 @@ class LmntTTSService(InterruptibleTTSService): logger.error(f"{self} error: {msg['error']}") await self.push_frame(TTSStoppedFrame()) await self.stop_all_metrics() - await self.push_error(ErrorFrame(f"{self} error: {msg['error']}")) + await self.push_error(ErrorFrame(error=f"{self} error: {msg['error']}")) return except json.JSONDecodeError: logger.error(f"Invalid JSON message: {message}") @@ -309,7 +311,8 @@ class LmntTTSService(InterruptibleTTSService): await self._get_websocket().send(json.dumps({"flush": True})) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} error sending message: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() @@ -317,3 +320,4 @@ class LmntTTSService(InterruptibleTTSService): yield None except Exception as e: logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index c0a6b6aaa..b8e984eeb 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -347,8 +347,8 @@ class MiniMaxHttpTTSService(TTSService): continue except Exception as e: - logger.exception(f"Error generating TTS: {e}") - yield ErrorFrame(error=f"MiniMax TTS error: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index 22a6e9999..992abbe67 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -294,7 +294,8 @@ class NeuphonicTTSService(InterruptibleTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} initialization error: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -307,7 +308,8 @@ class NeuphonicTTSService(InterruptibleTTSService): logger.debug("Disconnecting from Neuphonic") await self._websocket.close() except Exception as e: - logger.error(f"{self} error closing websocket: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._started = False self._websocket = None @@ -372,7 +374,8 @@ class NeuphonicTTSService(InterruptibleTTSService): await self._send_text(text) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} error sending message: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() @@ -380,6 +383,7 @@ class NeuphonicTTSService(InterruptibleTTSService): yield None except Exception as e: logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") class NeuphonicHttpTTSService(TTSService): @@ -582,7 +586,8 @@ class NeuphonicHttpTTSService(TTSService): yield TTSAudioRawFrame(audio_bytes, self.sample_rate, 1) except Exception as e: - logger.error(f"Error processing SSE message: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") # Don't yield error frame for individual message failures continue @@ -590,8 +595,8 @@ class NeuphonicHttpTTSService(TTSService): logger.debug("TTS generation cancelled") raise except Exception as e: - logger.exception(f"Error in run_tts: {e}") - yield ErrorFrame(error=f"Neuphonic TTS error: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index e38836b90..f66e6e8e1 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -478,7 +478,7 @@ class OpenAIRealtimeLLMService(LLMService): # it is to recover from a send-side error with proper state management, and that exponential # backoff for retries can have cost/stability implications for a service cluster, let's just # treat a send-side error as fatal. - await self.push_error(ErrorFrame(error=f"Error sending client event: {e}", fatal=True)) + await self.push_error(ErrorFrame(error=f"Error sending client event: {e}")) async def _update_settings(self): settings = self._session_properties @@ -667,9 +667,7 @@ class OpenAIRealtimeLLMService(LLMService): self._current_assistant_response = None # error handling if evt.response.status == "failed": - await self.push_error( - ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True) - ) + await self.push_error(ErrorFrame(error=evt.response.status_details["error"]["message"])) return # response content for item in evt.response.output: @@ -763,7 +761,7 @@ class OpenAIRealtimeLLMService(LLMService): async def _handle_evt_error(self, evt): # Errors are fatal to this connection. Send an ErrorFrame. - await self.push_error(ErrorFrame(error=f"Error: {evt}", fatal=True)) + await self.push_error(ErrorFrame(error=f"Error: {evt}")) # # state and client events for the current conversation diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index af580e4a0..a5231170e 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -199,7 +199,7 @@ class OpenAITTSService(TTSService): f"{self} error getting audio (status: {r.status_code}, error: {error})" ) yield ErrorFrame( - f"Error getting audio (status: {r.status_code}, error: {error})" + error=f"Error getting audio (status: {r.status_code}, error: {error})" ) return @@ -216,3 +216,4 @@ class OpenAITTSService(TTSService): yield TTSStoppedFrame() except BadRequestError as e: logger.exception(f"{self} error generating TTS: {e}") + yield ErrorFrame(error=f"{self} error: {e}") diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 922f9a572..af0600882 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -454,7 +454,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): # it is to recover from a send-side error with proper state management, and that exponential # backoff for retries can have cost/stability implications for a service cluster, let's just # treat a send-side error as fatal. - await self.push_error(ErrorFrame(error=f"Error sending client event: {e}", fatal=True)) + await self.push_error(ErrorFrame(error=f"Error sending client event: {e}")) async def _update_settings(self): settings = self._session_properties @@ -627,9 +627,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._current_assistant_response = None # error handling if evt.response.status == "failed": - await self.push_error( - ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True) - ) + await self.push_error(ErrorFrame(error=evt.response.status_details["error"]["message"])) return # response content for item in evt.response.output: @@ -687,7 +685,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_evt_error(self, evt): # Errors are fatal to this connection. Send an ErrorFrame. - await self.push_error(ErrorFrame(error=f"Error: {evt}", fatal=True)) + await self.push_error(ErrorFrame(error=f"Error: {evt}")) async def _handle_assistant_output(self, output): # We haven't seen intermixed audio and function_call items in the same response. But let's diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index 73addb9d1..3752418b5 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -101,7 +101,7 @@ class PiperTTSService(TTSService): f"{self} error getting audio (status: {response.status}, error: {error})" ) yield ErrorFrame( - f"Error getting audio (status: {response.status}, error: {error})" + error=f"Error getting audio (status: {response.status}, error: {error})" ) return @@ -117,8 +117,8 @@ class PiperTTSService(TTSService): await self.stop_ttfb_metrics() yield frame except Exception as e: - logger.error(f"Error in run_tts: {e}") - yield ErrorFrame(error=str(e)) + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") finally: logger.debug(f"{self}: Finished TTS [{text}]") await self.stop_ttfb_metrics() diff --git a/src/pipecat/services/playht/tts.py b/src/pipecat/services/playht/tts.py index 1f0ed2156..2c40065aa 100644 --- a/src/pipecat/services/playht/tts.py +++ b/src/pipecat/services/playht/tts.py @@ -266,7 +266,8 @@ class PlayHTTTSService(InterruptibleTTSService): self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") except Exception as e: - logger.error(f"{self} initialization error: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -279,7 +280,8 @@ class PlayHTTTSService(InterruptibleTTSService): logger.debug("Disconnecting from PlayHT") await self._websocket.close() except Exception as e: - logger.error(f"{self} error closing websocket: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._request_id = None self._websocket = None @@ -350,7 +352,7 @@ class PlayHTTTSService(InterruptibleTTSService): self._request_id = None elif "error" in msg: logger.error(f"{self} error: {msg}") - await self.push_error(ErrorFrame(f"{self} error: {msg['error']}")) + await self.push_error(ErrorFrame(error=f"{self} error: {msg['error']}")) except json.JSONDecodeError: logger.error(f"Invalid JSON message: {message}") @@ -392,7 +394,8 @@ class PlayHTTTSService(InterruptibleTTSService): await self._get_websocket().send(json.dumps(tts_command)) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} error sending message: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() @@ -402,8 +405,8 @@ class PlayHTTTSService(InterruptibleTTSService): yield None except Exception as e: - logger.error(f"{self} error generating TTS: {e}") - yield ErrorFrame(f"{self} error: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") class PlayHTHttpTTSService(TTSService): @@ -623,7 +626,8 @@ class PlayHTHttpTTSService(TTSService): yield frame except Exception as e: - logger.error(f"{self} error generating TTS: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index b51f9e3ec..329aecd7d 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -259,7 +259,8 @@ class RimeTTSService(AudioContextWordTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} initialization error: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -271,7 +272,8 @@ class RimeTTSService(AudioContextWordTTSService): await self._websocket.send(json.dumps(self._build_eos_msg())) await self._websocket.close() except Exception as e: - logger.error(f"{self} error closing websocket: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._context_id = None self._websocket = None @@ -367,7 +369,7 @@ class RimeTTSService(AudioContextWordTTSService): logger.error(f"{self} error: {msg}") await self.push_frame(TTSStoppedFrame()) await self.stop_all_metrics() - await self.push_error(ErrorFrame(f"{self} error: {msg['message']}")) + await self.push_error(ErrorFrame(error=f"{self} error: {msg['message']}")) self._context_id = None async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): @@ -409,7 +411,8 @@ class RimeTTSService(AudioContextWordTTSService): await self._get_websocket().send(json.dumps(msg)) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} error sending message: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() @@ -417,6 +420,7 @@ class RimeTTSService(AudioContextWordTTSService): yield None except Exception as e: logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") class RimeHttpTTSService(TTSService): @@ -574,8 +578,8 @@ class RimeHttpTTSService(TTSService): yield frame except Exception as e: - logger.exception(f"Error generating TTS: {e}") - yield ErrorFrame(error=f"Rime TTS error: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/riva/stt.py b/src/pipecat/services/riva/stt.py index 0c93365d5..25cc78c7a 100644 --- a/src/pipecat/services/riva/stt.py +++ b/src/pipecat/services/riva/stt.py @@ -659,8 +659,8 @@ class RivaSegmentedSTTService(SegmentedSTTService): yield ErrorFrame(f"Unexpected Riva response format: {str(ae)}") except Exception as e: - logger.exception(f"Riva Canary ASR error: {e}") - yield ErrorFrame(f"Riva Canary ASR error: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") class ParakeetSTTService(RivaSTTService): diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py index d051965ed..07aa4d105 100644 --- a/src/pipecat/services/riva/tts.py +++ b/src/pipecat/services/riva/tts.py @@ -23,6 +23,7 @@ from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( + ErrorFrame, Frame, TTSAudioRawFrame, TTSStartedFrame, @@ -165,6 +166,7 @@ class RivaTTSService(TTSService): add_response(None) except Exception as e: logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") add_response(None) await self.start_ttfb_metrics() diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index 9ff037938..6ce45d27d 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -264,7 +264,7 @@ class SarvamHttpTTSService(TTSService): if response.status != 200: error_text = await response.text() logger.error(f"Sarvam API error: {error_text}") - await self.push_error(ErrorFrame(f"Sarvam API error: {error_text}")) + await self.push_error(ErrorFrame(error=f"Sarvam API error: {error_text}")) return response_data = await response.json() @@ -274,7 +274,7 @@ class SarvamHttpTTSService(TTSService): # Decode base64 audio data if "audios" not in response_data or not response_data["audios"]: logger.error("No audio data received from Sarvam API") - await self.push_error(ErrorFrame("No audio data received")) + await self.push_error(ErrorFrame(error="No audio data received")) return # Get the first audio (there should be only one for single text input) @@ -296,7 +296,7 @@ class SarvamHttpTTSService(TTSService): except Exception as e: logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(f"Error generating TTS: {e}")) + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() @@ -578,7 +578,8 @@ class SarvamTTSService(InterruptibleTTSService): await self._disconnect_websocket() except Exception as e: - logger.error(f"Error during disconnect: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: # Reset state only after everything is cleaned up self._started = False @@ -602,7 +603,8 @@ class SarvamTTSService(InterruptibleTTSService): await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} initialization error: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -618,8 +620,8 @@ class SarvamTTSService(InterruptibleTTSService): await self._websocket.send(json.dumps(config_message)) logger.debug("Configuration sent successfully") except Exception as e: - logger.error(f"Failed to send config: {str(e)}") - await self.push_frame(ErrorFrame(f"Failed to send config: {str(e)}")) + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) raise async def _disconnect_websocket(self): @@ -632,6 +634,7 @@ class SarvamTTSService(InterruptibleTTSService): await self._websocket.close() except Exception as e: logger.error(f"{self} error closing websocket: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._started = False self._websocket = None @@ -661,7 +664,7 @@ class SarvamTTSService(InterruptibleTTSService): if "too long" in error_msg.lower() or "timeout" in error_msg.lower(): logger.warning("Connection timeout detected, service may need restart") - await self.push_frame(ErrorFrame(f"TTS Error: {error_msg}")) + await self.push_frame(ErrorFrame(error=f"TTS Error: {error_msg}")) async def _keepalive_task_handler(self): """Handle keepalive messages to maintain WebSocket connection.""" @@ -717,7 +720,8 @@ class SarvamTTSService(InterruptibleTTSService): await self._send_text(text) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} error sending message: {e}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() @@ -725,3 +729,4 @@ class SarvamTTSService(InterruptibleTTSService): yield None except Exception as e: logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") diff --git a/src/pipecat/services/soniox/stt.py b/src/pipecat/services/soniox/stt.py index 1447774e1..b4bcb7ba1 100644 --- a/src/pipecat/services/soniox/stt.py +++ b/src/pipecat/services/soniox/stt.py @@ -327,8 +327,8 @@ class SonioxSTTService(STTService): # Expected when closing the connection logger.debug("WebSocket connection closed, keepalive task stopped.") except Exception as e: - logger.error(f"{self} error (_keepalive_task_handler): {e}") - await self.push_error(ErrorFrame(f"{self} error (_keepalive_task_handler): {e}")) + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) async def _receive_task_handler(self): if not self._websocket: @@ -409,7 +409,7 @@ class SonioxSTTService(STTService): ) await self.push_error( ErrorFrame( - f"{self} error: {error_code} (_receive_task_handler) - {error_message}" + error=f"{self} error: {error_code} (_receive_task_handler) - {error_message}" ) ) @@ -425,5 +425,5 @@ class SonioxSTTService(STTService): # Expected when closing the connection. pass except Exception as e: - logger.error(f"{self} error: {e}") - await self.push_error(ErrorFrame(f"{self} error: {e}")) + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index f85660c83..2d95bd69e 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -467,8 +467,8 @@ class SpeechmaticsSTTService(STTService): await self._client.send_audio(audio) yield None except Exception as e: - logger.error(f"Speechmatics error: {e}") - yield ErrorFrame(f"Speechmatics error: {e}", fatal=False) + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") await self._disconnect() def update_params( @@ -514,6 +514,8 @@ class SpeechmaticsSTTService(STTService): self._client.send_message(payload), self.get_event_loop() ) except Exception as e: + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) raise RuntimeError(f"error sending message to STT: {e}") async def _connect(self) -> None: @@ -579,7 +581,8 @@ class SpeechmaticsSTTService(STTService): logger.debug(f"{self} Connected to Speechmatics STT service") await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} Error connecting to Speechmatics: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) self._client = None async def _disconnect(self) -> None: @@ -593,7 +596,8 @@ class SpeechmaticsSTTService(STTService): except asyncio.TimeoutError: logger.warning(f"{self} Timeout while closing Speechmatics client connection") except Exception as e: - logger.error(f"{self} Error closing Speechmatics client: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: self._client = None await self._call_event_handler("on_disconnected") diff --git a/src/pipecat/services/ultravox/stt.py b/src/pipecat/services/ultravox/stt.py index 987593f02..14eaebf6a 100644 --- a/src/pipecat/services/ultravox/stt.py +++ b/src/pipecat/services/ultravox/stt.py @@ -246,7 +246,8 @@ class UltravoxSTTService(AIService): logger.info("Model warm-up completed successfully") except Exception as e: - logger.warning(f"Model warm-up failed: {e}") + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) def _generate_silent_audio(self, sample_rate=16000, duration_sec=1.0): """Generate silent audio as a numpy array. @@ -376,7 +377,7 @@ class UltravoxSTTService(AIService): if arr.size > 0: # Check if array is not empty audio_arrays.append(arr) except Exception as e: - logger.error(f"Error processing bytes audio frame: {e}") + yield ErrorFrame(error=f"{self} error: {e}") # Handle numpy array data elif isinstance(f.audio, np.ndarray): if f.audio.size > 0: # Check if array is not empty @@ -436,14 +437,14 @@ class UltravoxSTTService(AIService): yield LLMFullResponseEndFrame() except Exception as e: - logger.error(f"Error generating text from model: {e}") - yield ErrorFrame(f"Error generating text: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") else: - logger.warning("No model available for text generation") + logger.error("No model available for text generation") yield ErrorFrame("No model available for text generation") except Exception as e: - logger.error(f"Error processing audio buffer: {e}") + logger.error(f"{self} exception: {e}") import traceback logger.error(traceback.format_exc()) diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index 17a911366..f9799b9c3 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -94,7 +94,7 @@ class WebsocketService(ABC): if self._reconnect_on_error: retry_count += 1 if retry_count >= MAX_RETRIES: - await report_error(ErrorFrame(message, fatal=True)) + await report_error(ErrorFrame(message)) break logger.warning(f"{self} connection error, will retry: {e}") diff --git a/src/pipecat/services/whisper/base_stt.py b/src/pipecat/services/whisper/base_stt.py index 026b14a63..743895253 100644 --- a/src/pipecat/services/whisper/base_stt.py +++ b/src/pipecat/services/whisper/base_stt.py @@ -226,8 +226,8 @@ class BaseWhisperSTTService(SegmentedSTTService): logger.warning("Received empty transcription from API") except Exception as e: - logger.exception(f"Exception during transcription: {e}") - yield ErrorFrame(f"Error during transcription: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") async def _transcribe(self, audio: bytes) -> Transcription: """Transcribe audio data to text. diff --git a/src/pipecat/services/whisper/stt.py b/src/pipecat/services/whisper/stt.py index e69ce39cd..4e326e2ca 100644 --- a/src/pipecat/services/whisper/stt.py +++ b/src/pipecat/services/whisper/stt.py @@ -428,5 +428,5 @@ class WhisperSTTServiceMLX(WhisperSTTService): ) except Exception as e: - logger.exception(f"MLX Whisper transcription error: {e}") - yield ErrorFrame(f"MLX Whisper transcription error: {str(e)}") + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") diff --git a/src/pipecat/services/xtts/tts.py b/src/pipecat/services/xtts/tts.py index df58c96e4..34ddecbe2 100644 --- a/src/pipecat/services/xtts/tts.py +++ b/src/pipecat/services/xtts/tts.py @@ -146,7 +146,7 @@ class XTTSService(TTSService): ) await self.push_error( ErrorFrame( - f"Error error getting studio speakers (status: {r.status}, error: {text})" + error=f"Error getting studio speakers (status: {r.status}, error: {text})" ) ) return @@ -187,7 +187,7 @@ class XTTSService(TTSService): if r.status != 200: text = await r.text() logger.error(f"{self} error getting audio (status: {r.status}, error: {text})") - yield ErrorFrame(f"Error getting audio (status: {r.status}, error: {text})") + yield ErrorFrame(error=f"Error getting audio (status: {r.status}, error: {text})") return await self.start_tts_usage_metrics(text) From 9bf88bbf14eb15e50e84b8603b57eebbeeaadb14 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 07:43:30 -0300 Subject: [PATCH 035/110] Fixing RivaTTSService error handler. --- src/pipecat/services/riva/tts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py index 07aa4d105..4d393943b 100644 --- a/src/pipecat/services/riva/tts.py +++ b/src/pipecat/services/riva/tts.py @@ -166,7 +166,6 @@ class RivaTTSService(TTSService): add_response(None) except Exception as e: logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") add_response(None) await self.start_ttfb_metrics() @@ -191,6 +190,7 @@ class RivaTTSService(TTSService): resp = await asyncio.wait_for(queue.get(), timeout=RIVA_TTS_TIMEOUT_SECS) except asyncio.TimeoutError: logger.error(f"{self} timeout waiting for audio response") + yield ErrorFrame(error=f"{self} error: {e}") await self.start_tts_usage_metrics(text) yield TTSStoppedFrame() From 7de8838debbc516cf18cad263375a6eeb339ab41 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Mon, 17 Nov 2025 13:25:12 +0100 Subject: [PATCH 036/110] add word-level timestamp support to Hume service --- src/pipecat/services/hume/tts.py | 45 +++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 6760c8121..b29215727 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -19,7 +19,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) -from pipecat.services.tts_service import TTSService +from pipecat.services.tts_service import WordTTSService from pipecat.utils.tracing.service_decorators import traced_tts try: @@ -28,6 +28,7 @@ try: FormatPcm, PostedUtterance, PostedUtteranceVoiceWithId, + TimestampType, ) except ModuleNotFoundError as e: # pragma: no cover - import-time guidance logger.error(f"Exception: {e}") @@ -38,7 +39,7 @@ except ModuleNotFoundError as e: # pragma: no cover - import-time guidance HUME_SAMPLE_RATE = 48_000 # Hume TTS streams at 48 kHz -class HumeTTSService(TTSService): +class HumeTTSService(WordTTSService): """Hume Octave Text-to-Speech service. Streams PCM audio via Hume's HTTP output streaming (JSON chunks) endpoint @@ -48,6 +49,7 @@ class HumeTTSService(TTSService): - Generates speech from text using Hume TTS. - Streams PCM audio. + - Supports word timestamps for synchronization with text. - Supports dynamic updates of voice and synthesis parameters at runtime. - Provides metrics for Time To First Byte (TTFB) and TTS usage. """ @@ -85,7 +87,9 @@ class HumeTTSService(TTSService): """ api_key = api_key or os.getenv("HUME_API_KEY") if not api_key: - raise ValueError("HumeTTSService requires an API key (env HUME_API_KEY or api_key=)") + raise ValueError( + "HumeTTSService requires an API key (env HUME_API_KEY or api_key=)" + ) if sample_rate != HUME_SAMPLE_RATE: logger.warning( @@ -101,6 +105,7 @@ class HumeTTSService(TTSService): self.set_voice(voice_id) self._audio_bytes = b"" + self._first_audio_chunk = True def can_generate_metrics(self) -> bool: """Can generate metrics. @@ -193,6 +198,7 @@ class HumeTTSService(TTSService): # Hume emits mono PCM at 48 kHz; downstream can resample if needed. # We buffer audio bytes before sending to prevent glitches. self._audio_bytes = b"" + self._first_audio_chunk = True # Use version "2" by default if no description is provided # Version "1" is needed when description is used @@ -202,11 +208,42 @@ class HumeTTSService(TTSService): format=pcm_fmt, instant_mode=True, version=version, + include_timestamp_types=[TimestampType.WORD], ): + # Check if this is a timestamp chunk + chunk_type = getattr(chunk, "type", None) + if chunk_type == "timestamp": + # Start word timestamps if we haven't received audio yet + if self._first_audio_chunk: + await self.stop_ttfb_metrics() + self.start_word_timestamps() + self._first_audio_chunk = False + # Process word timestamp + timestamp = getattr(chunk, "timestamp", None) + if timestamp: + word_text = getattr(timestamp, "text", None) + time_obj = getattr(timestamp, "time", None) + if word_text and time_obj: + # Convert milliseconds to seconds + begin_ms = getattr(time_obj, "begin", None) + if begin_ms is not None: + begin_seconds = begin_ms / 1000.0 + await self.add_word_timestamps( + [(word_text, begin_seconds)] + ) + continue + + # Process audio chunk audio_b64 = getattr(chunk, "audio", None) if not audio_b64: continue + # Start word timestamps on first audio chunk + if self._first_audio_chunk: + await self.stop_ttfb_metrics() + self.start_word_timestamps() + self._first_audio_chunk = False + pcm_bytes = base64.b64decode(audio_b64) self._audio_bytes += pcm_bytes @@ -230,4 +267,6 @@ class HumeTTSService(TTSService): finally: # Ensure TTFB timer is stopped even on early failures await self.stop_ttfb_metrics() + # Signal end of word timestamps + await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)]) yield TTSStoppedFrame() From 2f2bde9856cc13b060cecd34a60cc75d106906ae Mon Sep 17 00:00:00 2001 From: ivaaan Date: Mon, 17 Nov 2025 13:40:03 +0100 Subject: [PATCH 037/110] add timestamps to example --- .../foundational/07ae-interruptible-hume.py | 49 +++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index 046f2d4c8..501eafad3 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -13,13 +13,17 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame +from pipecat.frames.frames import Frame, LLMRunFrame, TTSTextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor +from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -28,9 +32,25 @@ from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.frames.frames import format_pts load_dotenv(override=True) + +class TimestampLogger(FrameProcessor): + """Frame processor that logs TTSTextFrame objects with their timestamps. + + This helps verify that word timestamps are working correctly by showing + when each word is spoken with its presentation timestamp (PTS). + """ + + async def process_frame(self, frame: Frame, direction: FrameDirection): + if isinstance(frame, TTSTextFrame): + pts_str = format_pts(frame.pts) if frame.pts else "no PTS" + logger.info(f"🎯 Word timestamp: '{frame.text}' at {pts_str}") + await self.push_frame(frame, direction) + + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -81,15 +101,24 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): rtvi = RTVIProcessor(config=RTVIConfig(config=[])) + # Add transcript processor to show timestamps in conversation history + transcript = TranscriptProcessor() + + # Add timestamp logger to verify word timestamps are being generated + timestamp_logger = TimestampLogger() + pipeline = Pipeline( [ transport.input(), # Transport user input rtvi, stt, + transcript.user(), # User transcripts context_aggregator.user(), # User responses llm, # LLM - tts, # TTS + tts, # TTS (HumeTTSService with word timestamps) + timestamp_logger, # Log word timestamps for verification transport.output(), # Transport bot output + transcript.assistant(), # Assistant transcripts with timestamps context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -109,11 +138,23 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_ready(rtvi): await rtvi.set_bot_ready() + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + """Log transcript updates to show timestamps in conversation.""" + for msg in frame.messages: + timestamp_str = f"[{msg.timestamp}] " if msg.timestamp else "" + logger.info(f"📝 Transcript: {timestamp_str}{msg.role}: {msg.content}") + @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") + logger.info( + "💡 Word timestamps are enabled! Watch for '🎯 Word timestamp' logs showing each word with its PTS." + ) # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + messages.append( + {"role": "system", "content": "Please introduce yourself to the user."} + ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") From 71869a116d06c2f120de059a242c319cbbbf8a0f Mon Sep 17 00:00:00 2001 From: ivaaan Date: Mon, 17 Nov 2025 13:51:04 +0100 Subject: [PATCH 038/110] fix errors --- examples/foundational/07ae-interruptible-hume.py | 4 ++++ src/pipecat/services/hume/tts.py | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index 501eafad3..17a29f0db 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -45,9 +45,13 @@ class TimestampLogger(FrameProcessor): """ async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TTSTextFrame): pts_str = format_pts(frame.pts) if frame.pts else "no PTS" logger.info(f"🎯 Word timestamp: '{frame.text}' at {pts_str}") + + # Always push all frames through await self.push_frame(frame, direction) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index b29215727..0ee6e2adf 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -28,7 +28,6 @@ try: FormatPcm, PostedUtterance, PostedUtteranceVoiceWithId, - TimestampType, ) except ModuleNotFoundError as e: # pragma: no cover - import-time guidance logger.error(f"Exception: {e}") @@ -208,7 +207,7 @@ class HumeTTSService(WordTTSService): format=pcm_fmt, instant_mode=True, version=version, - include_timestamp_types=[TimestampType.WORD], + include_timestamp_types=["word"], ): # Check if this is a timestamp chunk chunk_type = getattr(chunk, "type", None) From 77cd106795778fa22bcee3ba64588e7e79b9529a Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 09:54:08 -0300 Subject: [PATCH 039/110] Extracted the logic for retrying connections, and create a new send_with_retry method inside WebSocketService. --- src/pipecat/services/websocket_service.py | 74 +++++++++++++++++------ 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index f9799b9c3..d19e6ad4d 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -36,6 +36,7 @@ class WebsocketService(ABC): """ self._websocket: Optional[websockets.WebSocketClientProtocol] = None self._reconnect_on_error = reconnect_on_error + self._reconnect_in_progress: bool = False # Add this flag async def _verify_connection(self) -> bool: """Verify the websocket connection is active and responsive. @@ -66,6 +67,59 @@ class WebsocketService(ABC): await self._connect_websocket() return await self._verify_connection() + async def _try_reconnect( + self, + max_retries: int = 3, + report_error: Optional[Callable[[ErrorFrame], Awaitable[None]]] = None, + ) -> bool: + # Prevent concurrent reconnection attempts + if self._reconnect_in_progress: + logger.warning(f"{self} reconnect attempt aborted: already in progress") + return False + + self._reconnect_in_progress = True + last_exception: Optional[Exception] = None + try: + for attempt in range(1, max_retries + 1): + try: + logger.warning(f"{self} reconnecting, attempt {attempt}") + if await self._reconnect_websocket(attempt): + logger.info(f"{self} reconnected successfully on attempt {attempt}") + return True + except Exception as e: + last_exception = e + logger.error(f"{self} reconnection attempt {attempt} failed: {e}") + if report_error: + await report_error( + ErrorFrame(f"{self} reconnection attempt {attempt} failed: {e}") + ) + wait_time = exponential_backoff_time(attempt) + await asyncio.sleep(wait_time) + fatal_msg = f"{self} failed to reconnect after {max_retries} attempts" + if last_exception: + fatal_msg += f": {last_exception}" + logger.error(fatal_msg) + if report_error: + await report_error(ErrorFrame(fatal_msg, fatal=True)) + return False + finally: + self._reconnect_in_progress = False + + async def send_with_retry(self, message, report_error: Callable[[ErrorFrame], Awaitable[None]]): + """Attempt to send a message, retrying after reconnect if necessary.""" + try: + await self._websocket.send(message) + except Exception as e: + logger.error(f"{self} send failed: {e}, will try to reconnect") + # Try to reconnect before retrying + success = await self._try_reconnect(report_error=report_error) + if success: + logger.info(f"{self} reconnected successfully, will retry send the message") + # trying to send the message one more time + await self._websocket.send(message) + else: + logger.error(f"{self} send failed; unable to reconnect") + async def _receive_task_handler(self, report_error: Callable[[ErrorFrame], Awaitable[None]]): """Handle websocket message receiving with automatic retry logic. @@ -76,13 +130,9 @@ class WebsocketService(ABC): Args: report_error: Callback function to report connection errors. """ - retry_count = 0 - MAX_RETRIES = 3 - while True: try: await self._receive_messages() - retry_count = 0 # Reset counter on successful message receive except ConnectionClosedOK as e: # Normal closure, don't retry logger.debug(f"{self} connection closed normally: {e}") @@ -92,21 +142,9 @@ class WebsocketService(ABC): logger.error(message) if self._reconnect_on_error: - retry_count += 1 - if retry_count >= MAX_RETRIES: - await report_error(ErrorFrame(message)) + success = await self._try_reconnect(report_error=report_error) + if not success: break - - logger.warning(f"{self} connection error, will retry: {e}") - await report_error(ErrorFrame(message)) - - try: - if await self._reconnect_websocket(retry_count): - retry_count = 0 # Reset counter on successful reconnection - wait_time = exponential_backoff_time(retry_count) - await asyncio.sleep(wait_time) - except Exception as reconnect_error: - logger.error(f"{self} reconnection failed: {reconnect_error}") else: await report_error(ErrorFrame(message)) break From 19cc0177b85bea279692abb1d829567667d73027 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 09:54:20 -0300 Subject: [PATCH 040/110] Refactored DeepgramFluxSTTService to automatically reconnect if sending a message fails. --- src/pipecat/services/deepgram/flux/stt.py | 114 ++++++++++++++++++---- 1 file changed, 95 insertions(+), 19 deletions(-) diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index ac6e8bb08..a0d45afe4 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -6,7 +6,9 @@ """Deepgram Flux speech-to-text service implementation.""" +import asyncio import json +import time from enum import Enum from typing import Any, AsyncGenerator, Dict, Optional from urllib.parse import urlencode @@ -94,6 +96,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): mip_opt_out: Optional. Opts out requests from the Deepgram Model Improvement Program (default False). tag: List of tags to label requests for identification during usage reporting. + min_confidence: Optional. Minimum confidence required confidence to create a TranscriptionFrame """ eager_eot_threshold: Optional[float] = None @@ -102,6 +105,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): keyterm: list = [] mip_opt_out: Optional[bool] = None tag: list = [] + min_confidence: Optional[float] = None # New parameter def __init__( self, @@ -163,6 +167,13 @@ class DeepgramFluxSTTService(WebsocketSTTService): self._register_event_handler("on_end_of_turn") self._register_event_handler("on_eager_end_of_turn") self._register_event_handler("on_update") + self._connection_established_event = asyncio.Event() + # Watchdog task to prevent dangling tasks + # If we stop sending audio to Flux after we have received that the User has started speaking + # we never receive the user stopped speaking event unless we resume sending audio to it. + self._last_stt_time = None + self._watchdog_task = None + self._user_is_speaking = False async def _connect(self): """Connect to WebSocket and start background tasks. @@ -172,9 +183,6 @@ class DeepgramFluxSTTService(WebsocketSTTService): """ await self._connect_websocket() - if self._websocket and not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) - async def _disconnect(self): """Disconnect from WebSocket and clean up tasks. @@ -182,14 +190,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): and cleans up resources to prevent memory leaks. """ try: - # Cancel background tasks BEFORE closing websocket - if self._receive_task: - await self.cancel_task(self._receive_task, timeout=2.0) - self._receive_task = None - - # Now close the websocket await self._disconnect_websocket() - except Exception as e: logger.error(f"{self} exception: {e}") await self.push_error(ErrorFrame(error=f"{self} error: {e}")) @@ -197,6 +198,25 @@ class DeepgramFluxSTTService(WebsocketSTTService): # Reset state only after everything is cleaned up self._websocket = None + async def _send_silence(self, duration_secs: float = 0.5): + """Send a block of silence of the specified duration (default 500 ms).""" + sample_width = 2 # bytes per sample for 16-bit PCM + num_channels = 1 # mono + num_samples = int(self.sample_rate * duration_secs) + silence = b"\x00" * (num_samples * sample_width * num_channels) + await self._websocket.send(silence) + + async def _watchdog_task_handler(self): + while self._websocket and self._websocket.state is State.OPEN: + now = time.monotonic() + # More than 500 ms without sending new audio to Flux + if self._user_is_speaking and self._last_stt_time and now - self._last_stt_time > 0.5: + logger.warning("Sending silence to Flux to prevent dangling task") + await self._send_silence() + self._last_stt_time = time.monotonic() + # check every 100ms + await asyncio.sleep(0.1) + async def _connect_websocket(self): """Establish WebSocket connection to API. @@ -208,10 +228,26 @@ class DeepgramFluxSTTService(WebsocketSTTService): if self._websocket and self._websocket.state is State.OPEN: return + self._connection_established_event.clear() + self._user_is_speaking = False self._websocket = await websocket_connect( self._websocket_url, additional_headers={"Authorization": f"Token {self._api_key}"}, ) + + # Creating the receiver task + if not self._receive_task: + self._receive_task = self.create_task( + self._receive_task_handler(self._report_error) + ) + + # Creating the watchdog task + if not self._watchdog_task: + self._watchdog_task = self.create_task(self._watchdog_task_handler()) + + # Now wait for the connection established event + logger.debug("WebSocket connected, waiting for server confirmation...") + await self._connection_established_event.wait() logger.debug("Connected to Deepgram Flux Websocket") await self._call_event_handler("on_connected") except Exception as e: @@ -227,6 +263,16 @@ class DeepgramFluxSTTService(WebsocketSTTService): metrics collection. Handles disconnection errors gracefully. """ try: + # Cancel background tasks BEFORE closing websocket + if self._receive_task: + await self.cancel_task(self._receive_task, timeout=2.0) + self._receive_task = None + if self._watchdog_task: + await self.cancel_task(self._watchdog_task, timeout=2.0) + self._watchdog_task = None + self._last_stt_time = None + + self._connection_established_event.clear() await self.stop_all_metrics() if self._websocket: @@ -340,7 +386,8 @@ class DeepgramFluxSTTService(WebsocketSTTService): return try: - await self._websocket.send(audio) + self._last_stt_time = time.monotonic() + await self.send_with_retry(audio, self._report_error) except Exception as e: logger.error(f"{self} exception: {e}") yield ErrorFrame(error=f"{self} error: {e}") @@ -463,6 +510,8 @@ class DeepgramFluxSTTService(WebsocketSTTService): transcription processing. """ logger.info("Connected to Flux - ready to stream audio") + # Notify connection is established + self._connection_established_event.set() async def _handle_fatal_error(self, data: Dict[str, Any]): """Handle fatal error messages from Deepgram Flux. @@ -530,6 +579,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): transcript: maybe the first few words of the turn. """ logger.debug("User started speaking") + self._user_is_speaking = True await self.push_interruption_task_frame_and_wait() await self.broadcast_frame(UserStartedSpeakingFrame) await self.start_metrics() @@ -550,6 +600,22 @@ class DeepgramFluxSTTService(WebsocketSTTService): logger.trace(f"Received event TurnResumed: {event}") await self._call_event_handler("on_turn_resumed") + def _calculate_average_confidence(self, transcript_data) -> Optional[float]: + """Calculate the average confidence from transcript data. + + Return None if the data is missing or invalid. + """ + # Example: Assume transcript_data has a list of words with confidence + words = transcript_data.get("words") + if not words or not isinstance(words, list): + return None + confidences = [ + w.get("confidence") for w in words if isinstance(w.get("confidence"), (float, int)) + ] + if not confidences: + return None + return sum(confidences) / len(confidences) + async def _handle_end_of_turn(self, transcript: str, data: Dict[str, Any]): """Handle EndOfTurn events from Deepgram Flux. @@ -569,16 +635,26 @@ class DeepgramFluxSTTService(WebsocketSTTService): data: The TurnInfo message data containing event type, transcript and some extra metadata. """ logger.debug("User stopped speaking") + self._user_is_speaking = False - await self.push_frame( - TranscriptionFrame( - transcript, - self._user_id, - time_now_iso8601(), - self._language, - result=data, + # Compute the average confidence + average_confidence = self._calculate_average_confidence(data) + + if not self._params.min_confidence or average_confidence > self._params.min_confidence: + await self.push_frame( + TranscriptionFrame( + transcript, + self._user_id, + time_now_iso8601(), + self._language, + result=data, + ) ) - ) + else: + logger.warning( + f"Transcription confidence below min_confidence threshold: {average_confidence}" + ) + await self._handle_transcription(transcript, True, self._language) await self.stop_processing_metrics() await self.push_frame(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM) From 04dbbabc036289fb3d90e5228a3e5af573d1ba21 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 09:54:30 -0300 Subject: [PATCH 041/110] Introduced a minimum confidence parameter in DeepgramFluxSTTService to avoid generating transcriptions below a defined threshold. --- examples/foundational/07c-interruptible-deepgram-flux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/foundational/07c-interruptible-deepgram-flux.py b/examples/foundational/07c-interruptible-deepgram-flux.py index f5b246acd..62579c2c5 100644 --- a/examples/foundational/07c-interruptible-deepgram-flux.py +++ b/examples/foundational/07c-interruptible-deepgram-flux.py @@ -52,7 +52,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramFluxSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramFluxSTTService( + api_key=os.getenv("DEEPGRAM_API_KEY"), + params=DeepgramFluxSTTService.InputParams(min_confidence=0.3), + ) tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-2-andromeda-en") From b104a59b109168c01c0240ae885f7b3a0fd7e2ce Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 09:54:39 -0300 Subject: [PATCH 042/110] Mentioning the Deepgram Flux improvements in the changelog. --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cda253c7..90a7542de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a watchdog to `DeepgramFluxSTTService` to prevent dangling tasks in case the + user was speaking and we stop receiving audio. + +- Introduced a minimum confidence parameter in `DeepgramFluxSTTService` to avoid + generating transcriptions below a defined threshold. + - Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT service from ElevenLabs. @@ -18,6 +24,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Extracted the logic for retrying connections, and create a new `send_with_retry` + method inside `WebSocketService`. + +- Refactored `DeepgramFluxSTTService` to automatically reconnect if sending a + message fails. + - Updated all STT and TTS services to use consistent error handling pattern with `push_error()` method for better pipeline error event integration. From 9156e217275684231a9d29c75750ea616744360e Mon Sep 17 00:00:00 2001 From: ivaaan Date: Mon, 17 Nov 2025 14:00:03 +0100 Subject: [PATCH 043/110] fix formatting --- examples/foundational/07ae-interruptible-hume.py | 7 ++----- src/pipecat/services/hume/tts.py | 8 ++------ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index 17a29f0db..e30cfff98 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -13,7 +13,7 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import Frame, LLMRunFrame, TTSTextFrame +from pipecat.frames.frames import Frame, LLMRunFrame, TTSTextFrame, format_pts from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -32,7 +32,6 @@ from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams -from pipecat.frames.frames import format_pts load_dotenv(override=True) @@ -156,9 +155,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): "💡 Word timestamps are enabled! Watch for '🎯 Word timestamp' logs showing each word with its PTS." ) # Kick off the conversation. - messages.append( - {"role": "system", "content": "Please introduce yourself to the user."} - ) + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 0ee6e2adf..c3ef18ddf 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -86,9 +86,7 @@ class HumeTTSService(WordTTSService): """ api_key = api_key or os.getenv("HUME_API_KEY") if not api_key: - raise ValueError( - "HumeTTSService requires an API key (env HUME_API_KEY or api_key=)" - ) + raise ValueError("HumeTTSService requires an API key (env HUME_API_KEY or api_key=)") if sample_rate != HUME_SAMPLE_RATE: logger.warning( @@ -227,9 +225,7 @@ class HumeTTSService(WordTTSService): begin_ms = getattr(time_obj, "begin", None) if begin_ms is not None: begin_seconds = begin_ms / 1000.0 - await self.add_word_timestamps( - [(word_text, begin_seconds)] - ) + await self.add_word_timestamps([(word_text, begin_seconds)]) continue # Process audio chunk From 2a51d0f1e5d4cb1546e214e3f3d75b9cacc8cd50 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Mon, 17 Nov 2025 15:20:06 +0100 Subject: [PATCH 044/110] add changelog --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cda253c7..baed81a18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 services that subclass `TTSService` can indicate whether the text in the `TTSTextFrame`s they push already contain any necessary inter-frame spaces. +- Added word-level timestamps support to Hume TTS service + ### Changed - Updated all STT and TTS services to use consistent error handling pattern with @@ -56,8 +58,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no ONNX dependency or added processing complexity. ## [0.0.94] - 2025-11-10 From f3b254e335be513623a6b30b80e1c60353d1f22e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 14 Nov 2025 21:39:18 -0500 Subject: [PATCH 045/110] D'oh! My TTS "inter-frame-spaces" logic was *way* overcomplicated (and fundamentally mistaken, though it happened to work) Now: - For TTS word-by-word output and `TTSSpeakFrames`: `TTSTextFrame`s' have `includes_inter_frame_spaces=False`. - For all other TTS output: `TTSTextFrame` pass through the received text frames' `includes_inter_frame_spaces` value. So far, this value has always been `True`: LLMs send text chunks already containing all necessary spaces. - `LLMTextFrame`s set `includes_inter_frame_spaces=False` at init time, per the aforementioned assumption. --- CHANGELOG.md | 8 +--- src/pipecat/frames/frames.py | 5 ++- src/pipecat/services/anthropic/llm.py | 4 +- src/pipecat/services/asyncai/tts.py | 18 -------- src/pipecat/services/aws/llm.py | 4 +- src/pipecat/services/aws/tts.py | 9 ---- src/pipecat/services/azure/tts.py | 9 ---- src/pipecat/services/deepgram/tts.py | 18 -------- src/pipecat/services/fish/tts.py | 9 ---- .../services/google/gemini_live/llm.py | 2 - src/pipecat/services/google/llm.py | 4 +- src/pipecat/services/google/tts.py | 18 -------- src/pipecat/services/groq/tts.py | 9 ---- src/pipecat/services/hume/tts.py | 9 ---- src/pipecat/services/inworld/tts.py | 9 ---- src/pipecat/services/lmnt/tts.py | 9 ---- src/pipecat/services/minimax/tts.py | 9 ---- src/pipecat/services/neuphonic/tts.py | 18 -------- src/pipecat/services/openai/base_llm.py | 4 +- src/pipecat/services/openai/realtime/llm.py | 2 - src/pipecat/services/openai/tts.py | 9 ---- src/pipecat/services/piper/tts.py | 9 ---- src/pipecat/services/rime/tts.py | 9 ---- src/pipecat/services/riva/tts.py | 9 ---- src/pipecat/services/sambanova/llm.py | 4 +- src/pipecat/services/sarvam/tts.py | 18 -------- src/pipecat/services/speechmatics/tts.py | 9 ---- src/pipecat/services/tts_service.py | 44 +++++++++---------- 28 files changed, 33 insertions(+), 255 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cda253c7..6a4a64f2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT service from ElevenLabs. -- Added a `TTSService.includes_inter_frame_spaces` property getter, so that TTS - services that subclass `TTSService` can indicate whether the text in the - `TTSTextFrame`s they push already contain any necessary inter-frame spaces. - ### Changed - Updated all STT and TTS services to use consistent error handling pattern with @@ -56,8 +52,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no ONNX dependency or added processing complexity. ## [0.0.94] - 2025-11-10 diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 6f48f79f7..ddb4e5a14 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -352,7 +352,10 @@ class TextFrame(DataFrame): class LLMTextFrame(TextFrame): """Text frame generated by LLM services.""" - pass + def __post_init__(self): + super().__post_init__() + # LLM services send text frames with all necessary spaces included + self.includes_inter_frame_spaces = True @dataclass diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index a5c9ee791..2e3e0272d 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -373,9 +373,7 @@ class AnthropicLLMService(LLMService): if event.type == "content_block_delta": if hasattr(event.delta, "text"): - frame = LLMTextFrame(event.delta.text) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self.push_frame(LLMTextFrame(event.delta.text)) completion_tokens_estimate += self._estimate_tokens(event.delta.text) elif hasattr(event.delta, "partial_json") and tool_use_block: json_accumulator += event.delta.partial_json diff --git a/src/pipecat/services/asyncai/tts.py b/src/pipecat/services/asyncai/tts.py index 8df597c56..f916d9bdd 100644 --- a/src/pipecat/services/asyncai/tts.py +++ b/src/pipecat/services/asyncai/tts.py @@ -146,15 +146,6 @@ class AsyncAITTSService(InterruptibleTTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that AsyncAI TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that AsyncAI's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Async language format. @@ -433,15 +424,6 @@ class AsyncAIHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that AsyncAI TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that AsyncAI's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Async language format. diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index 147a8c12a..ccbac43b7 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -1078,9 +1078,7 @@ class AWSBedrockLLMService(LLMService): if "contentBlockDelta" in event: delta = event["contentBlockDelta"]["delta"] if "text" in delta: - frame = LLMTextFrame(delta["text"]) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self.push_frame(LLMTextFrame(delta["text"])) completion_tokens_estimate += self._estimate_tokens(delta["text"]) elif "toolUse" in delta and "input" in delta["toolUse"]: # Handle partial JSON for tool use diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index cbc35b123..f22c42399 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -209,15 +209,6 @@ class AWSPollyTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that AWS TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that AWS's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to AWS Polly language format. diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index 8ac8b70e3..a1040f312 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -151,15 +151,6 @@ class AzureBaseTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Azure TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Azure's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Azure language format. diff --git a/src/pipecat/services/deepgram/tts.py b/src/pipecat/services/deepgram/tts.py index f6045c1f3..f75d40b09 100644 --- a/src/pipecat/services/deepgram/tts.py +++ b/src/pipecat/services/deepgram/tts.py @@ -79,15 +79,6 @@ class DeepgramTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Deepgram TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Deepgram's text frames include necessary inter-frame spaces. - """ - return True - @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Deepgram's TTS API. @@ -177,15 +168,6 @@ class DeepgramHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Deepgram TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Deepgram's text frames include necessary inter-frame spaces. - """ - return True - @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Deepgram's TTS API. diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index 0acb12a96..5fe129998 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -159,15 +159,6 @@ class FishAudioTTSService(InterruptibleTTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Fish Audio TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Fish Audio's text frames include necessary inter-frame spaces. - """ - return True - async def set_model(self, model: str): """Set the TTS model and reconnect. diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 11632968e..7e0b0f494 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -1452,8 +1452,6 @@ class GeminiLiveLLMService(LLMService): self._bot_text_buffer += text self._search_result_buffer += text # Also accumulate for grounding frame = LLMTextFrame(text=text) - # Gemini Live text already includes any necessary inter-chunk spaces - frame.includes_inter_frame_spaces = True await self.push_frame(frame) # Check for grounding metadata in server content diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index ad5fd70a7..883932b76 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -920,9 +920,7 @@ class GoogleLLMService(LLMService): for part in candidate.content.parts: if not part.thought and part.text: search_result += part.text - frame = LLMTextFrame(part.text) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self.push_frame(LLMTextFrame(part.text)) elif part.function_call: function_call = part.function_call id = function_call.id or str(uuid.uuid4()) diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index 449b2bca2..cf03e2d52 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -596,15 +596,6 @@ class GoogleHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Google TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Google's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Google TTS language format. @@ -803,15 +794,6 @@ class GoogleBaseTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Google and Gemini TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Google's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Google TTS language format. diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index d2dcb73a1..9026c4c4c 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -111,15 +111,6 @@ class GroqTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Groq TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Groq's text frames include necessary inter-frame spaces. - """ - return True - @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Groq's TTS API. diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 6760c8121..a3a7e9a4c 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -110,15 +110,6 @@ class HumeTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Hume TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Hume's text frames include necessary inter-frame spaces. - """ - return True - async def start(self, frame: StartFrame) -> None: """Start the service. diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index 067ea6ab6..dc2282b91 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -250,15 +250,6 @@ class InworldTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Inworld TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Inworld's text frames include necessary inter-frame spaces. - """ - return True - async def start(self, frame: StartFrame): """Start the Inworld TTS service. diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index d98d3f76c..ebcad0f20 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -124,15 +124,6 @@ class LmntTTSService(InterruptibleTTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that LMNT TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that LMNT's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to LMNT service language format. diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index b8e984eeb..05e2dac3b 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -194,15 +194,6 @@ class MiniMaxHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that MiniMax TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that MiniMax's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to MiniMax service language format. diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index 992abbe67..60b0ebcb1 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -151,15 +151,6 @@ class NeuphonicTTSService(InterruptibleTTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Neuphonic TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Neuphonic's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Neuphonic service language format. @@ -449,15 +440,6 @@ class NeuphonicHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Neuphonic TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Neuphonic's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Neuphonic service language format. diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 5a8c1ab31..d020e1106 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -390,9 +390,7 @@ class BaseOpenAILLMService(LLMService): # Keep iterating through the response to collect all the argument fragments arguments += tool_call.function.arguments elif chunk.choices[0].delta.content: - frame = LLMTextFrame(chunk.choices[0].delta.content) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self.push_frame(LLMTextFrame(chunk.choices[0].delta.content)) # When gpt-4o-audio / gpt-4o-mini-audio is used for llm or stt+llm # we need to get LLMTextFrame for the transcript diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index f66e6e8e1..8eaa3d6fa 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -678,8 +678,6 @@ class OpenAIRealtimeLLMService(LLMService): # the output modality is "text" if evt.delta: frame = LLMTextFrame(evt.delta) - # OpenAI Realtime text already includes any necessary inter-chunk spaces - frame.includes_inter_frame_spaces = True await self.push_frame(frame) async def _handle_evt_audio_transcript_delta(self, evt): diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index a5231170e..23cb75324 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -131,15 +131,6 @@ class OpenAITTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that OpenAI TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that OpenAI's text frames include necessary inter-frame spaces. - """ - return True - async def set_model(self, model: str): """Set the TTS model to use. diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index 3752418b5..dd842ff11 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -66,15 +66,6 @@ class PiperTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Piper TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Piper's text frames include necessary inter-frame spaces. - """ - return True - @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Piper's HTTP API. diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 329aecd7d..7b62f20fa 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -501,15 +501,6 @@ class RimeHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Rime TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Rime's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> str | None: """Convert pipecat language to Rime language code. diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py index 07aa4d105..6de6f9332 100644 --- a/src/pipecat/services/riva/tts.py +++ b/src/pipecat/services/riva/tts.py @@ -113,15 +113,6 @@ class RivaTTSService(TTSService): riva.client.proto.riva_tts_pb2.RivaSynthesisConfigRequest() ) - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Riva TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Riva's text frames include necessary inter-frame spaces. - """ - return True - async def set_model(self, model: str): """Attempt to set the TTS model. diff --git a/src/pipecat/services/sambanova/llm.py b/src/pipecat/services/sambanova/llm.py index 76f11e81c..5ed600457 100644 --- a/src/pipecat/services/sambanova/llm.py +++ b/src/pipecat/services/sambanova/llm.py @@ -176,9 +176,7 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore # Keep iterating through the response to collect all the argument fragments arguments += tool_call.function.arguments elif chunk.choices[0].delta.content: - frame = LLMTextFrame(chunk.choices[0].delta.content) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self.push_frame(LLMTextFrame(chunk.choices[0].delta.content)) # When gpt-4o-audio / gpt-4o-mini-audio is used for llm or stt+llm # we need to get LLMTextFrame for the transcript diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index 6ce45d27d..127a0d589 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -195,15 +195,6 @@ class SarvamHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Sarvam TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Sarvam's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Sarvam AI language format. @@ -467,15 +458,6 @@ class SarvamTTSService(InterruptibleTTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Sarvam TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Sarvam's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Sarvam AI language format. diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index e115d5a7c..b8fe172e7 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -105,15 +105,6 @@ class SpeechmaticsTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Speechmatics TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Speechmatics's text frames include necessary inter-frame spaces. - """ - return True - @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Speechmatics' HTTP API. diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 29c54f497..f0d602a40 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -142,6 +142,7 @@ class TTSService(AIService): self._voice_id: str = "" self._settings: Dict[str, Any] = {} self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() + self._aggregated_text_includes_inter_frame_spaces: bool = False self._text_filters: Sequence[BaseTextFilter] = text_filters or [] self._transport_destination: Optional[str] = transport_destination self._tracing_enabled: bool = False @@ -192,23 +193,6 @@ class TTSService(AIService): CHUNK_SECONDS = 0.5 return int(self.sample_rate * CHUNK_SECONDS * 2) # 2 bytes/sample - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates whether TTSTextFrames include necesary inter-frame spaces. - - When True, the TTSTextFrame objects pushed by this service already - include all necessary spaces between subsequent frames. When False, - downstream processors (like the assistant context aggregator) may need - to add spacing. - - Subclasses should override this property to return True if their text - generation process already includes necessary inter-frame spaces. - - Returns: - False by default. Subclasses can override to return True. - """ - return False - async def set_model(self, model: str): """Set the TTS model to use. @@ -369,9 +353,16 @@ class TTSService(AIService): await self._maybe_pause_frame_processing() sentence = self._text_aggregator.text + includes_inter_frame_spaces = self._aggregated_text_includes_inter_frame_spaces + + # Reset aggregator state await self._text_aggregator.reset() self._processing_text = False - await self._push_tts_frames(sentence) + self._aggregated_text_includes_inter_frame_spaces = False + + await self._push_tts_frames( + sentence, includes_inter_frame_spaces=includes_inter_frame_spaces + ) if isinstance(frame, LLMFullResponseEndFrame): if self._push_text_frames: await self.push_frame(frame, direction) @@ -380,7 +371,8 @@ class TTSService(AIService): elif isinstance(frame, TTSSpeakFrame): # Store if we were processing text or not so we can set it back. processing_text = self._processing_text - await self._push_tts_frames(frame.text) + # Assumption: text in TTSSpeakFrame does not include inter-frame spaces + await self._push_tts_frames(frame.text, includes_inter_frame_spaces=False) # We pause processing incoming frames because we are sending data to # the TTS. We pause to avoid audio overlapping. await self._maybe_pause_frame_processing() @@ -474,11 +466,17 @@ class TTSService(AIService): text = frame.text else: text = await self._text_aggregator.aggregate(frame.text) + # Assumption: whether inter-frame spaces are included shouldn't + # change during aggregation, so we can just use the latest frame's + # value + self._aggregated_text_includes_inter_frame_spaces = frame.includes_inter_frame_spaces if text: - await self._push_tts_frames(text) + await self._push_tts_frames( + text, includes_inter_frame_spaces=frame.includes_inter_frame_spaces + ) - async def _push_tts_frames(self, text: str): + async def _push_tts_frames(self, text: str, includes_inter_frame_spaces: bool): # Remove leading newlines only text = text.lstrip("\n") @@ -508,7 +506,7 @@ class TTSService(AIService): # We send the original text after the audio. This way, if we are # interrupted, the text is not added to the assistant context. frame = TTSTextFrame(text) - frame.includes_inter_frame_spaces = self.includes_inter_frame_spaces + frame.includes_inter_frame_spaces = includes_inter_frame_spaces await self.push_frame(frame) async def _stop_frame_handler(self): @@ -635,6 +633,8 @@ class WordTTSService(TTSService): frame = TTSStoppedFrame() frame.pts = last_pts else: + # Assumption: word-by-word text frames don't include spaces, so + # we can rely on the default includes_inter_frame_spaces=False frame = TTSTextFrame(word) frame.pts = self._initial_word_timestamp + timestamp if frame: From 0c3c26b7b87863ecc79f148ad394adc4f939a31b Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 15:20:09 -0300 Subject: [PATCH 046/110] Passing the custom request_data to the SmallWebRTCRunnerArguments body. --- src/pipecat/runner/run.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index 55c70ed8a..ebca467df 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -264,7 +264,10 @@ def _setup_webrtc_routes( # Prepare runner arguments with the callback to run your bot async def webrtc_connection_callback(connection): bot_module = _get_bot_module() - runner_args = SmallWebRTCRunnerArguments(webrtc_connection=connection) + + runner_args = SmallWebRTCRunnerArguments( + webrtc_connection=connection, body=request.request_data + ) background_tasks.add_task(bot_module.bot, runner_args) # Delegate handling to SmallWebRTCRequestHandler @@ -326,7 +329,8 @@ def _setup_webrtc_routes( type=request_data["type"], pc_id=request_data.get("pc_id"), restart_pc=request_data.get("restart_pc"), - request_data=request_data, + request_data=request_data.get("request_data") + or request_data.get("requestData"), ) return await offer(webrtc_request, background_tasks) elif request.method == HTTPMethod.PATCH.value: From 74154b26a2a97840bd60a4f8ddb676da7a26f77f Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 15:39:07 -0300 Subject: [PATCH 047/110] Mentioning the SmallWebRTCTransport fix in the readme. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33f7fdc6b..bca4a5d6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue in the `Runner` where, when using `SmallWebRTCTransport`, the + `request_data` was not being passed to the `SmallWebRTCRunnerArguments` body. + - Fixed subtle issue of assistant context messages ending up with double spaces between words or sentences. From 7eedb33d5076de8584452dd311c844bb46122cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 17 Nov 2025 11:19:04 -0800 Subject: [PATCH 048/110] BaseTextFilter: only require subclasses to implement filter() --- CHANGELOG.md | 12 ++++++------ src/pipecat/utils/text/base_text_filter.py | 3 --- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bca4a5d6f..d4fce0917 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT service from ElevenLabs. +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no + ONNX dependency or added processing complexity. + ### Changed +- `BaseTextFilter` only require subclasses to implement the `filter()` method. + - Extracted the logic for retrying connections, and create a new `send_with_retry` method inside `WebSocketService`. @@ -65,12 +71,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. -### Added - -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no - ONNX dependency or added processing complexity. - ## [0.0.94] - 2025-11-10 ### Changed diff --git a/src/pipecat/utils/text/base_text_filter.py b/src/pipecat/utils/text/base_text_filter.py index 1a18a38a6..0ede2137c 100644 --- a/src/pipecat/utils/text/base_text_filter.py +++ b/src/pipecat/utils/text/base_text_filter.py @@ -26,7 +26,6 @@ class BaseTextFilter(ABC): behavior, settings management, and interruption handling logic. """ - @abstractmethod async def update_settings(self, settings: Mapping[str, Any]): """Update the filter's configuration settings. @@ -53,7 +52,6 @@ class BaseTextFilter(ABC): """ pass - @abstractmethod async def handle_interruption(self): """Handle interruption events in the processing pipeline. @@ -62,7 +60,6 @@ class BaseTextFilter(ABC): """ pass - @abstractmethod async def reset_interruption(self): """Reset the filter state after an interruption has been handled. From 5095fc6a64da534abfedab0c438761288547087e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Nov 2025 15:16:19 -0500 Subject: [PATCH 049/110] Update Moondream example so that Moondream service output makes it into the context, even if the TTS service is disabled --- .../14d-function-calling-moondream-video.py | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/examples/foundational/14d-function-calling-moondream-video.py b/examples/foundational/14d-function-calling-moondream-video.py index 6aeb2b892..9544818b9 100644 --- a/examples/foundational/14d-function-calling-moondream-video.py +++ b/examples/foundational/14d-function-calling-moondream-video.py @@ -15,14 +15,21 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame +from pipecat.frames.frames import ( + Frame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMRunFrame, + TextFrame, + UserImageRequestFrame, +) from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.processors.frame_processor import FrameDirection +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -66,6 +73,27 @@ async def fetch_user_image(params: FunctionCallParams): # await params.result_callback({"result": "Image is being captured."}) +class MoondreamTextFrameWrapper(FrameProcessor): + """Wraps Moondream-provided TextFrames with LLM response start/end frames. + + This processor detects TextFrames and automatically wraps them with + LLMFullResponseStartFrame and LLMFullResponseEndFrame to provide proper + response boundaries for downstream processors. + """ + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + # If we receive a TextFrame, wrap it with response start/end frames + if isinstance(frame, TextFrame): + await self.push_frame(LLMFullResponseStartFrame(), direction) + await self.push_frame(frame, direction) + await self.push_frame(LLMFullResponseEndFrame(), direction) + else: + # For all other frames, just pass them through + await self.push_frame(frame, direction) + + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -130,6 +158,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # If you run into weird description, try with use_cpu=True moondream = MoondreamService() + # Wrap TextFrames with LLM response start/end frames, which makes Moondream + # output be treated like LLM responses for the purpose of context + # aggregation. Without this, the assistant context aggregator would ignore + # Moondream output (if the TTS service is disabled). + moondream_text_wrapper = MoondreamTextFrameWrapper() + pipeline = Pipeline( [ transport.input(), # Transport user input @@ -137,7 +171,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context_aggregator.user(), # User responses ParallelPipeline( [llm], # LLM - [moondream], + [moondream, moondream_text_wrapper], ), tts, # TTS transport.output(), # Transport bot output From 4835617b16a1f4a5fbbb2f70562454c75a568573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Nov 2025 15:20:01 -0800 Subject: [PATCH 050/110] ConsumerProcessor: queue frames internally instead of pushing them --- CHANGELOG.md | 4 ++++ src/pipecat/processors/consumer_processor.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4fce0917..04d789370 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `ConsumerProcessor` now queues frames from the producer internally instead of + pushing them directly. This allows us to subclass consumer processors and + manipulate frames before they are pushed. + - `BaseTextFilter` only require subclasses to implement the `filter()` method. - Extracted the logic for retrying connections, and create a new `send_with_retry` diff --git a/src/pipecat/processors/consumer_processor.py b/src/pipecat/processors/consumer_processor.py index 5445b492d..3654194ec 100644 --- a/src/pipecat/processors/consumer_processor.py +++ b/src/pipecat/processors/consumer_processor.py @@ -83,4 +83,4 @@ class ConsumerProcessor(FrameProcessor): while True: frame = await self._queue.get() new_frame = await self._transformer(frame) - await self.push_frame(new_frame, self._direction) + await self.queue_frame(new_frame, self._direction) From 3132e1226571228bda6fc6a323128a2c3095e003 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 14 Nov 2025 10:48:52 -0500 Subject: [PATCH 051/110] Add camera and screen capture support to dev runner for SmallWebRTC --- CHANGELOG.md | 8 ++++++++ src/pipecat/runner/utils.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04d789370..de1f520a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no + ONNX dependency or added processing complexity. + - Added a watchdog to `DeepgramFluxSTTService` to prevent dangling tasks in case the user was speaking and we stop receiving audio. @@ -39,6 +43,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated all STT and TTS services to use consistent error handling pattern with `push_error()` method for better pipeline error event integration. +- Added support for `maybe_capture_participant_camera()` and + `maybe_capture_participant_screen()` for `SmallWebRTCTransport` in the runner + utils. + - Added Hindi support for Rime TTS services. - Updated `GeminiTTSService` to use Google Cloud Text-to-Speech streaming API diff --git a/src/pipecat/runner/utils.py b/src/pipecat/runner/utils.py index f9fd0c14a..76a6fa82f 100644 --- a/src/pipecat/runner/utils.py +++ b/src/pipecat/runner/utils.py @@ -281,6 +281,14 @@ async def maybe_capture_participant_camera( except ImportError: pass + try: + from pipecat.transports.smallwebrtc.transport import SmallWebRTCTransport + + if isinstance(transport, SmallWebRTCTransport): + await transport.capture_participant_video(video_source="camera") + except ImportError: + pass + async def maybe_capture_participant_screen( transport: BaseTransport, client: Any, framerate: int = 0 @@ -303,6 +311,14 @@ async def maybe_capture_participant_screen( except ImportError: pass + try: + from pipecat.transports.smallwebrtc.transport import SmallWebRTCTransport + + if isinstance(transport, SmallWebRTCTransport): + await transport.capture_participant_video(video_source="screenVideo") + except ImportError: + pass + def _smallwebrtc_sdp_cleanup_ice_candidates(text: str, pattern: str) -> str: """Clean up ICE candidates in SDP text for SmallWebRTC. From a510b276e6ba1dbb678f4bd6b3b17fbdc61e409b Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 18 Nov 2025 11:37:57 -0300 Subject: [PATCH 052/110] Ensure that the function call results respect the previous LLM context. --- CHANGELOG.md | 5 ++ .../aggregators/llm_response_universal.py | 74 ++++++++++++------- 2 files changed, 54 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de1f520a7..f2fa5801e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a race condition where, if the LLM received instructions to both produce + text and invoke a function call at the same time, the context would not be + updated before the function call result arrived, causing the bot to repeat + itself. + - Fixed an issue in the `Runner` where, when using `SmallWebRTCTransport`, the `request_data` was not being passed to the `SmallWebRTCRunnerArguments` body. diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index d4d9ad7da..98d7514b6 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -594,6 +594,8 @@ class LLMAssistantAggregator(LLMContextAggregator): self._started = 0 self._function_calls_in_progress: Dict[str, Optional[FunctionCallInProgressFrame]] = {} self._context_updated_tasks: Set[asyncio.Task] = set() + self._function_calls_context_messages = [] + self._function_calls_pending_context_updates_callbacks = [] @property def has_function_calls_in_progress(self) -> bool: @@ -650,21 +652,23 @@ class LLMAssistantAggregator(LLMContextAggregator): async def push_aggregation(self): """Push the current assistant aggregation with timestamp.""" - if not self._aggregation: - return + if self._aggregation: + aggregation = self.aggregation_string() + await self.reset() - aggregation = self.aggregation_string() - await self.reset() + if aggregation: + self._context.add_message({"role": "assistant", "content": aggregation}) - if aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) + # Push context frame + await self.push_context_frame() - # Push context frame - await self.push_context_frame() + # Push timestamp frame with current time + timestamp_frame = LLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) - # Push timestamp frame with current time - timestamp_frame = LLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) + if self._function_calls_context_messages: + self._flush_function_call_messages_to_context() + await self.push_context_frame(FrameDirection.UPSTREAM) async def _handle_llm_run(self, frame: LLMRunFrame): await self.push_context_frame(FrameDirection.UPSTREAM) @@ -684,6 +688,23 @@ class LLMAssistantAggregator(LLMContextAggregator): self._started = 0 await self.reset() + def _flush_function_call_messages_to_context(self): + """Move all function calls messages into context, then clear the list.""" + if self._function_calls_context_messages: + self._context.add_messages(self._function_calls_context_messages) + self._function_calls_context_messages.clear() + + # Call the `on_context_updated` callbacks once the function call results + # are added to the context. Run them in separate tasks to make + # sure we don't block the pipeline. + for callback, task_name in self._function_calls_pending_context_updates_callbacks: + task = self.create_task(callback(), task_name) + self._context_updated_tasks.add(task) + task.add_done_callback(self._context_updated_task_finished) + + # Clear the pending callbacks list + self._function_calls_pending_context_updates_callbacks.clear() + async def _handle_function_calls_started(self, frame: FunctionCallsStartedFrame): function_names = [f"{f.function_name}:{f.tool_call_id}" for f in frame.function_calls] logger.debug(f"{self} FunctionCallsStartedFrame: {function_names}") @@ -696,7 +717,7 @@ class LLMAssistantAggregator(LLMContextAggregator): ) # Update context with the in-progress function call - self._context.add_message( + self._function_calls_context_messages.append( { "role": "assistant", "tool_calls": [ @@ -711,7 +732,7 @@ class LLMAssistantAggregator(LLMContextAggregator): ], } ) - self._context.add_message( + self._function_calls_context_messages.append( { "role": "tool", "content": "IN_PROGRESS", @@ -742,6 +763,13 @@ class LLMAssistantAggregator(LLMContextAggregator): else: self._update_function_call_result(frame.function_name, frame.tool_call_id, "COMPLETED") + # Store the on_context_updated callback along with task name info to be invoked later + if properties and properties.on_context_updated: + task_name = f"{frame.function_name}:{frame.tool_call_id}:on_context_updated" + self._function_calls_pending_context_updates_callbacks.append( + (properties.on_context_updated, task_name) + ) + run_llm = False # Run inference if the function call result requires it. @@ -756,17 +784,13 @@ class LLMAssistantAggregator(LLMContextAggregator): # If this is the last function call in progress, run the LLM. run_llm = not bool(self._function_calls_in_progress) - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) - - # Call the `on_context_updated` callback once the function call result - # is added to the context. Also, run this in a separate task to make - # sure we don't block the pipeline. - if properties and properties.on_context_updated: - task_name = f"{frame.function_name}:{frame.tool_call_id}:on_context_updated" - task = self.create_task(properties.on_context_updated(), task_name) - self._context_updated_tasks.add(task) - task.add_done_callback(self._context_updated_task_finished) + # Only run if the LLM response has completed (not currently generating), + # otherwise defer execution until push_aggregation() is called + # (triggered by LLMFullResponseEndFrame or interruption). + if not self._started: + self._flush_function_call_messages_to_context() + if run_llm: + await self.push_context_frame(FrameDirection.UPSTREAM) async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame): logger.debug( @@ -781,7 +805,7 @@ class LLMAssistantAggregator(LLMContextAggregator): del self._function_calls_in_progress[frame.tool_call_id] def _update_function_call_result(self, function_name: str, tool_call_id: str, result: Any): - for message in self._context.get_messages(): + for message in self._function_calls_context_messages: if ( not isinstance(message, LLMSpecificMessage) and message["role"] == "tool" From fccc91e923fc16617c45c1e001a086c99cfc6d78 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 18 Nov 2025 11:50:28 -0300 Subject: [PATCH 053/110] Searching in both _function_calls_context_messages and context messages when updating the result. --- .../processors/aggregators/llm_response_universal.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 98d7514b6..4e9f19ff3 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -805,7 +805,12 @@ class LLMAssistantAggregator(LLMContextAggregator): del self._function_calls_in_progress[frame.tool_call_id] def _update_function_call_result(self, function_name: str, tool_call_id: str, result: Any): - for message in self._function_calls_context_messages: + def iter_all(): + yield from self._function_calls_context_messages + # In case on long-running function call, the function may already be added to the context + yield from self._context.get_messages() + + for message in iter_all(): if ( not isinstance(message, LLMSpecificMessage) and message["role"] == "tool" From 9f45ad4d2e2ec9713a95c9d7c39b2bba5dd258cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Nov 2025 08:38:45 -0800 Subject: [PATCH 054/110] LLMContext: create_image_message/create_audio_message are now async --- CHANGELOG.md | 5 ++ .../foundational/12-describe-image-openai.py | 2 +- .../12a-describe-image-anthropic.py | 2 +- .../foundational/12b-describe-image-aws.py | 2 +- .../12c-describe-image-gemini-flash.py | 2 +- .../processors/aggregators/llm_context.py | 46 ++++++++++++------- 6 files changed, 38 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de1f520a7..82da28b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- ⚠️ Breaking change: `LLMContext.create_image_message()` and + `LLMContext.create_audio_message()` are now async methods. This fixes and + issue where the asyncio event loop would be blocked while encoding audio or + images. + - `ConsumerProcessor` now queues frames from the producer internally instead of pushing them directly. This allows us to subclass consumer processors and manipulate frames before they are pushed. diff --git a/examples/foundational/12-describe-image-openai.py b/examples/foundational/12-describe-image-openai.py index 8c72075e8..477803da6 100644 --- a/examples/foundational/12-describe-image-openai.py +++ b/examples/foundational/12-describe-image-openai.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Kick off the conversation. image = Image.open(image_path) - message = LLMContext.create_image_message( + message = await LLMContext.create_image_message( image=image.tobytes(), format="RGB", size=image.size, diff --git a/examples/foundational/12a-describe-image-anthropic.py b/examples/foundational/12a-describe-image-anthropic.py index 6a9891712..ac4e8f01c 100644 --- a/examples/foundational/12a-describe-image-anthropic.py +++ b/examples/foundational/12a-describe-image-anthropic.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Kick off the conversation. image = Image.open(image_path) - message = LLMContext.create_image_message( + message = await LLMContext.create_image_message( image=image.tobytes(), format="RGB", size=image.size, diff --git a/examples/foundational/12b-describe-image-aws.py b/examples/foundational/12b-describe-image-aws.py index 441c49cfd..cf1ce66a0 100644 --- a/examples/foundational/12b-describe-image-aws.py +++ b/examples/foundational/12b-describe-image-aws.py @@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Kick off the conversation. image = Image.open(image_path) - message = LLMContext.create_image_message( + message = await LLMContext.create_image_message( image=image.tobytes(), format="RGB", size=image.size, diff --git a/examples/foundational/12c-describe-image-gemini-flash.py b/examples/foundational/12c-describe-image-gemini-flash.py index 919bf3553..bfd7f5146 100644 --- a/examples/foundational/12c-describe-image-gemini-flash.py +++ b/examples/foundational/12c-describe-image-gemini-flash.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Kick off the conversation. image = Image.open(image_path) - message = LLMContext.create_image_message( + message = await LLMContext.create_image_message( image=image.tobytes(), format="RGB", size=image.size, diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index d9280f9c0..b9216103a 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -14,6 +14,7 @@ translation from this universal context into whatever format it needs, using a service-specific adapter. """ +import asyncio import base64 import io import wave @@ -137,7 +138,7 @@ class LLMContext: return {"role": role, "content": content} @staticmethod - def create_image_message( + async def create_image_message( *, role: str = "user", format: str, @@ -154,15 +155,21 @@ class LLMContext: image: Raw image bytes. text: Optional text to include with the image. """ - buffer = io.BytesIO() - Image.frombytes(format, size, image).save(buffer, format="JPEG") - encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") + + def encode_image(): + buffer = io.BytesIO() + Image.frombytes(format, size, image).save(buffer, format="JPEG") + encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") + return encoded_image + + encoded_image = await asyncio.to_thread(encode_image) + url = f"data:image/jpeg;base64,{encoded_image}" return LLMContext.create_image_url_message(role=role, url=url, text=text) @staticmethod - def create_audio_message( + async def create_audio_message( *, role: str = "user", audio_frames: list[AudioRawFrame], text: str = "Audio follows" ) -> LLMContextMessage: """Create a context message containing audio. @@ -172,21 +179,26 @@ class LLMContext: audio_frames: List of audio frame objects to include. text: Optional text to include with the audio. """ - sample_rate = audio_frames[0].sample_rate - num_channels = audio_frames[0].num_channels - content = [] - content.append({"type": "text", "text": text}) - data = b"".join(frame.audio for frame in audio_frames) + def encode_audio(): + sample_rate = audio_frames[0].sample_rate + num_channels = audio_frames[0].num_channels - with io.BytesIO() as buffer: - with wave.open(buffer, "wb") as wf: - wf.setsampwidth(2) - wf.setnchannels(num_channels) - wf.setframerate(sample_rate) - wf.writeframes(data) + content = [] + content.append({"type": "text", "text": text}) + data = b"".join(frame.audio for frame in audio_frames) - encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") + with io.BytesIO() as buffer: + with wave.open(buffer, "wb") as wf: + wf.setsampwidth(2) + wf.setnchannels(num_channels) + wf.setframerate(sample_rate) + wf.writeframes(data) + + encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") + return encoded_audio + + encoded_audio = asyncio.to_thread(encode_audio) content.append( { From 9944e6faf0447f3034e5a3c7e95db0ef1ec387e5 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 18:25:53 +0100 Subject: [PATCH 055/110] upd service based on Mark's suggestions --- src/pipecat/services/hume/tts.py | 126 ++++++++++++++++++++----------- 1 file changed, 81 insertions(+), 45 deletions(-) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index c3ef18ddf..027a5851b 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -14,11 +14,13 @@ from pydantic import BaseModel from pipecat.frames.frames import ( ErrorFrame, Frame, + InterruptionFrame, StartFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.processors.frame_processor import FrameDirection from pipecat.services.tts_service import WordTTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -29,6 +31,7 @@ try: PostedUtterance, PostedUtteranceVoiceWithId, ) + from hume.tts.types import TimestampMessage except ModuleNotFoundError as e: # pragma: no cover - import-time guidance logger.error(f"Exception: {e}") logger.error("In order to use Hume, you need to `pip install pipecat-ai[hume]`.") @@ -48,7 +51,7 @@ class HumeTTSService(WordTTSService): - Generates speech from text using Hume TTS. - Streams PCM audio. - - Supports word timestamps for synchronization with text. + - Supports word-level timestamps for precise audio-text synchronization. - Supports dynamic updates of voice and synthesis parameters at runtime. - Provides metrics for Time To First Byte (TTFB) and TTS usage. """ @@ -93,7 +96,12 @@ class HumeTTSService(WordTTSService): f"Hume TTS streams at {HUME_SAMPLE_RATE} Hz; configured sample_rate={sample_rate}" ) - super().__init__(sample_rate=sample_rate, **kwargs) + # WordTTSService sets push_text_frames=False by default, which we want + super().__init__( + sample_rate=sample_rate, + push_text_frames=False, + **kwargs, + ) self._client = AsyncHumeClient(api_key=api_key) self._params = params or HumeTTSService.InputParams() @@ -102,7 +110,10 @@ class HumeTTSService(WordTTSService): self.set_voice(voice_id) self._audio_bytes = b"" - self._first_audio_chunk = True + + # Track cumulative time for word timestamps across utterances + self._cumulative_time = 0.0 + self._started = False def can_generate_metrics(self) -> bool: """Can generate metrics. @@ -128,6 +139,27 @@ class HumeTTSService(WordTTSService): frame: The start frame. """ await super().start(frame) + self._reset_state() + + def _reset_state(self): + """Reset internal state variables.""" + self._cumulative_time = 0.0 + self._started = False + + async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame and handle state changes. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ + await super().push_frame(frame, direction) + if isinstance(frame, (InterruptionFrame, TTSStoppedFrame)): + # Reset timing on interruption or stop + self._reset_state() + + if isinstance(frame, TTSStoppedFrame): + await self.add_word_timestamps([("Reset", 0)]) async def update_setting(self, key: str, value: Any) -> None: """Runtime updates via `TTSUpdateSettingsFrame`. @@ -144,7 +176,7 @@ class HumeTTSService(WordTTSService): if key_l == "voice_id": self.set_voice(str(value)) - logger.info(f"HumeTTSService voice_id set to: {self.voice}") + logger.debug(f"HumeTTSService voice_id set to: {self.voice}") elif key_l == "description": self._params.description = None if value is None else str(value) elif key_l == "speed": @@ -157,7 +189,7 @@ class HumeTTSService(WordTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate speech from text using Hume TTS. + """Generate speech from text using Hume TTS with word timestamps. Args: text: The text to be synthesized. @@ -188,64 +220,66 @@ class HumeTTSService(WordTTSService): await self.start_ttfb_metrics() await self.start_tts_usage_metrics(text) - yield TTSStartedFrame() + + # Start TTS sequence if not already started + if not self._started: + self.start_word_timestamps() + yield TTSStartedFrame() + self._started = True try: # Instant mode is always enabled here (not user-configurable) # Hume emits mono PCM at 48 kHz; downstream can resample if needed. # We buffer audio bytes before sending to prevent glitches. self._audio_bytes = b"" - self._first_audio_chunk = True # Use version "2" by default if no description is provided # Version "1" is needed when description is used version = "1" if self._params.description is not None else "2" + + # Track the duration of this utterance based on the last timestamp + utterance_duration = 0.0 + async for chunk in self._client.tts.synthesize_json_streaming( utterances=[utterance], format=pcm_fmt, instant_mode=True, version=version, - include_timestamp_types=["word"], + include_timestamp_types=["word"], # Request word-level timestamps ): - # Check if this is a timestamp chunk - chunk_type = getattr(chunk, "type", None) - if chunk_type == "timestamp": - # Start word timestamps if we haven't received audio yet - if self._first_audio_chunk: - await self.stop_ttfb_metrics() - self.start_word_timestamps() - self._first_audio_chunk = False - # Process word timestamp - timestamp = getattr(chunk, "timestamp", None) - if timestamp: - word_text = getattr(timestamp, "text", None) - time_obj = getattr(timestamp, "time", None) - if word_text and time_obj: - # Convert milliseconds to seconds - begin_ms = getattr(time_obj, "begin", None) - if begin_ms is not None: - begin_seconds = begin_ms / 1000.0 - await self.add_word_timestamps([(word_text, begin_seconds)]) - continue - - # Process audio chunk + # Process audio chunks audio_b64 = getattr(chunk, "audio", None) - if not audio_b64: - continue - - # Start word timestamps on first audio chunk - if self._first_audio_chunk: + if audio_b64: await self.stop_ttfb_metrics() - self.start_word_timestamps() - self._first_audio_chunk = False + pcm_bytes = base64.b64decode(audio_b64) + self._audio_bytes += pcm_bytes - pcm_bytes = base64.b64decode(audio_b64) - self._audio_bytes += pcm_bytes + # Buffer audio until we have enough to avoid glitches + if len(self._audio_bytes) >= self.chunk_size: + frame = TTSAudioRawFrame( + audio=self._audio_bytes, + sample_rate=self.sample_rate, + num_channels=1, + ) + yield frame + self._audio_bytes = b"" - # Buffer audio until we have enough to avoid glitches - if len(self._audio_bytes) < self.chunk_size: - continue + # Process timestamp messages + if isinstance(chunk, TimestampMessage): + timestamp = chunk.timestamp + if timestamp.type == "word": + # Convert milliseconds to seconds and add cumulative offset + word_start_time = self._cumulative_time + (timestamp.time.begin / 1000.0) + word_end_time = self._cumulative_time + (timestamp.time.end / 1000.0) + # Track the maximum end time for this utterance + utterance_duration = max(utterance_duration, word_end_time) + + # Add word timestamp + await self.add_word_timestamps([(timestamp.text, word_start_time)]) + + # Flush any remaining audio bytes + if self._audio_bytes: frame = TTSAudioRawFrame( audio=self._audio_bytes, sample_rate=self.sample_rate, @@ -256,12 +290,14 @@ class HumeTTSService(WordTTSService): self._audio_bytes = b"" + # Update cumulative time for next utterance + if utterance_duration > 0: + self._cumulative_time = utterance_duration + except Exception as e: logger.error(f"{self} exception: {e}") await self.push_error(ErrorFrame(error=f"{self} error: {e}")) finally: # Ensure TTFB timer is stopped even on early failures await self.stop_ttfb_metrics() - # Signal end of word timestamps - await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)]) - yield TTSStoppedFrame() + # Let the parent class handle TTSStoppedFrame via push_stop_frames From 26f96d0be8e8d1813157bb92b0b99c1b04535a28 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 18:31:38 +0100 Subject: [PATCH 056/110] upd example --- .../foundational/07ae-interruptible-hume.py | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index e30cfff98..eae9edaa9 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -13,7 +13,8 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import Frame, LLMRunFrame, TTSTextFrame, format_pts +from pipecat.frames.frames import LLMRunFrame, TTSTextFrame +from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -21,7 +22,6 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, ) -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments @@ -29,6 +29,7 @@ from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.hume.tts import HUME_SAMPLE_RATE, HumeTTSService from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -36,24 +37,6 @@ from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams load_dotenv(override=True) -class TimestampLogger(FrameProcessor): - """Frame processor that logs TTSTextFrame objects with their timestamps. - - This helps verify that word timestamps are working correctly by showing - when each word is spoken with its presentation timestamp (PTS). - """ - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if isinstance(frame, TTSTextFrame): - pts_str = format_pts(frame.pts) if frame.pts else "no PTS" - logger.info(f"🎯 Word timestamp: '{frame.text}' at {pts_str}") - - # Always push all frames through - await self.push_frame(frame, direction) - - # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -107,9 +90,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Add transcript processor to show timestamps in conversation history transcript = TranscriptProcessor() - # Add timestamp logger to verify word timestamps are being generated - timestamp_logger = TimestampLogger() - pipeline = Pipeline( [ transport.input(), # Transport user input @@ -119,7 +99,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context_aggregator.user(), # User responses llm, # LLM tts, # TTS (HumeTTSService with word timestamps) - timestamp_logger, # Log word timestamps for verification transport.output(), # Transport bot output transcript.assistant(), # Assistant transcripts with timestamps context_aggregator.assistant(), # Assistant spoken responses @@ -134,7 +113,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): audio_out_sample_rate=HUME_SAMPLE_RATE, ), idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - observers=[RTVIObserver(rtvi)], + observers=[ + RTVIObserver(rtvi), + DebugLogObserver( + frame_types={ + TTSTextFrame: (BaseOutputTransport, FrameEndpoint.DESTINATION), + } + ), + ], ) @rtvi.event_handler("on_client_ready") @@ -152,7 +138,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_connected(transport, client): logger.info(f"Client connected") logger.info( - "💡 Word timestamps are enabled! Watch for '🎯 Word timestamp' logs showing each word with its PTS." + "💡 Word timestamps are enabled! Watch the console for TTSTextFrame logs showing each word with its PTS." ) # Kick off the conversation. messages.append({"role": "system", "content": "Please introduce yourself to the user."}) From 153201542bacd6a524627d20a93717a3782fdb08 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 18 Nov 2025 13:29:06 -0500 Subject: [PATCH 057/110] Fix foundational 30 example to output TTSTextFrames synced to audio --- examples/foundational/30-observer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index c41177df2..5a01c2934 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -150,7 +150,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): LLMLogObserver(), DebugLogObserver( frame_types={ - TTSTextFrame: (BaseOutputTransport, FrameEndpoint.DESTINATION), + TTSTextFrame: (BaseOutputTransport, FrameEndpoint.SOURCE), UserStartedSpeakingFrame: (BaseInputTransport, FrameEndpoint.SOURCE), EndFrame: None, } From a38f20813505665c03f07b15d8f7ad07d4e36b8b Mon Sep 17 00:00:00 2001 From: Ivan A Date: Tue, 18 Nov 2025 20:30:28 +0100 Subject: [PATCH 058/110] Update examples/foundational/07ae-interruptible-hume.py Co-authored-by: Mark Backman --- examples/foundational/07ae-interruptible-hume.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index eae9edaa9..9233e58dd 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): RTVIObserver(rtvi), DebugLogObserver( frame_types={ - TTSTextFrame: (BaseOutputTransport, FrameEndpoint.DESTINATION), + TTSTextFrame: (BaseOutputTransport, FrameEndpoint.SOURCE), } ), ], From 4ae1819645f2b6ef4b9f925ded629946b962d90e Mon Sep 17 00:00:00 2001 From: Ivan A Date: Tue, 18 Nov 2025 20:30:44 +0100 Subject: [PATCH 059/110] Update src/pipecat/services/hume/tts.py Co-authored-by: Mark Backman --- src/pipecat/services/hume/tts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index bba70a759..278626748 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -100,6 +100,7 @@ class HumeTTSService(WordTTSService): super().__init__( sample_rate=sample_rate, push_text_frames=False, + push_stop_frames=True, **kwargs, ) From c2309efd7eea91779ad4e8c829e8237ca88eb17e Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 20:32:27 +0100 Subject: [PATCH 060/110] rm TranscriptProcessor --- examples/foundational/07ae-interruptible-hume.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index 9233e58dd..a7bea804c 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -23,7 +23,6 @@ from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, ) from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor -from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -87,9 +86,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): rtvi = RTVIProcessor(config=RTVIConfig(config=[])) - # Add transcript processor to show timestamps in conversation history - transcript = TranscriptProcessor() - pipeline = Pipeline( [ transport.input(), # Transport user input From 4c3fd42b1ccea1be1ff6153398bb5b2e199dd953 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 20:36:45 +0100 Subject: [PATCH 061/110] fix changelog --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83796864b..b346e002d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,8 +90,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. -### Added - - Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and example wiring; leverages the enhancement model for robust detection with no ONNX dependency or added processing complexity. From f325eeb95baf01b8cf54421c0e21dde932fb85ad Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 20:41:10 +0100 Subject: [PATCH 062/110] rm TranscriptProcessor 2 --- examples/foundational/07ae-interruptible-hume.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ae-interruptible-hume.py index a7bea804c..c5de34c85 100644 --- a/examples/foundational/07ae-interruptible-hume.py +++ b/examples/foundational/07ae-interruptible-hume.py @@ -91,12 +91,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): transport.input(), # Transport user input rtvi, stt, - transcript.user(), # User transcripts context_aggregator.user(), # User responses llm, # LLM tts, # TTS (HumeTTSService with word timestamps) transport.output(), # Transport bot output - transcript.assistant(), # Assistant transcripts with timestamps context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -123,13 +121,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_ready(rtvi): await rtvi.set_bot_ready() - @transcript.event_handler("on_transcript_update") - async def on_transcript_update(processor, frame): - """Log transcript updates to show timestamps in conversation.""" - for msg in frame.messages: - timestamp_str = f"[{msg.timestamp}] " if msg.timestamp else "" - logger.info(f"📝 Transcript: {timestamp_str}{msg.role}: {msg.content}") - @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") From 3d21faaac264197fdc6f6558cf5ef31da29083e0 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 18 Nov 2025 11:55:36 -0500 Subject: [PATCH 063/110] `LLMAssistantAggregator` now properly aggregates text that might be a mix of `includes_inter_frame_spaces=True` and `includes_inter_frame_spaces=False` frames --- .../aggregators/llm_response_universal.py | 31 ++++---- .../processors/transcript_processor.py | 22 +++--- src/pipecat/utils/string.py | 70 +++++++++++++++++-- tests/test_context_aggregators.py | 50 +++++++++++++ 4 files changed, 136 insertions(+), 37 deletions(-) diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index d4d9ad7da..d0ac67257 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -66,7 +66,7 @@ from pipecat.processors.aggregators.llm_response import ( LLMUserAggregatorParams, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.utils.string import concatenate_aggregated_text +from pipecat.utils.string import TextPartForConcatenation, concatenate_aggregated_text from pipecat.utils.time import time_now_iso8601 @@ -90,15 +90,7 @@ class LLMContextAggregator(FrameProcessor): self._context = context self._role = role - self._aggregation: List[str] = [] - - # Whether to add spaces between text parts. - # (Currently only used by LLMAssistantAggregator, but could be expanded - # to LLMUserAggregator in the future if needed; that would require - # additional work since LLMUserAggregator currently trims spaces from - # incoming frames before determining whether it "really" received any - # text). - self._add_spaces = True + self._aggregation: List[TextPartForConcatenation] = [] @property def messages(self) -> List[LLMContextMessage]: @@ -191,7 +183,7 @@ class LLMContextAggregator(FrameProcessor): Returns: The concatenated aggregation string. """ - return concatenate_aggregated_text(self._aggregation, self._add_spaces) + return concatenate_aggregated_text(self._aggregation) class LLMUserAggregator(LLMContextAggregator): @@ -441,7 +433,12 @@ class LLMUserAggregator(LLMContextAggregator): if not text.strip(): return - self._aggregation.append(text) + # Transcriptions never include inter-part spaces (so far). + self._aggregation.append( + TextPartForConcatenation( + text, includes_inter_part_spaces=frame.includes_inter_frame_spaces + ) + ) # We just got a final result, so let's reset interim results. self._seen_interim_results = False # Reset aggregation timer. @@ -821,11 +818,11 @@ class LLMAssistantAggregator(LLMContextAggregator): if len(frame.text) == 0: return - # Track whether we need to add spaces between text parts - # Assumption: we can just keep track of the latest frame's value - self._add_spaces = not frame.includes_inter_frame_spaces - - self._aggregation.append(frame.text) + self._aggregation.append( + TextPartForConcatenation( + frame.text, includes_inter_part_spaces=frame.includes_inter_frame_spaces + ) + ) def _context_updated_task_finished(self, task: asyncio.Task): self._context_updated_tasks.discard(task) diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 0b6f1b5bb..93e0c37b4 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -26,7 +26,7 @@ from pipecat.frames.frames import ( TTSTextFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.utils.string import concatenate_aggregated_text +from pipecat.utils.string import TextPartForConcatenation, concatenate_aggregated_text from pipecat.utils.time import time_now_iso8601 @@ -98,15 +98,9 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): **kwargs: Additional arguments passed to parent class. """ super().__init__(**kwargs) - self._current_text_parts: List[str] = [] + self._current_text_parts: List[TextPartForConcatenation] = [] self._aggregation_start_time: Optional[str] = None - # Whether to add spaces between text parts. - # (The use of this could be expanded to the UserTranscriptProcessor in - # the future if needed; currently the UserTranscriptProcessor assumes - # that user transcription frames do not need aggregation). - self._add_spaces = True - async def _emit_aggregated_text(self): """Aggregates and emits text fragments as a transcript message. @@ -147,7 +141,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): Result: "Hello there how are you" """ if self._current_text_parts and self._aggregation_start_time: - content = concatenate_aggregated_text(self._current_text_parts, self._add_spaces) + content = concatenate_aggregated_text(self._current_text_parts) if content: logger.trace(f"Emitting aggregated assistant message: {content}") message = TranscriptionMessage( @@ -191,11 +185,11 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): if not self._aggregation_start_time: self._aggregation_start_time = time_now_iso8601() - # Track whether we need to add spaces between text parts - # Assumption: we can just keep track of the latest frame's value - self._add_spaces = not frame.includes_inter_frame_spaces - - self._current_text_parts.append(frame.text) + self._current_text_parts.append( + TextPartForConcatenation( + frame.text, includes_inter_part_spaces=frame.includes_inter_frame_spaces + ) + ) # Push frame. await self.push_frame(frame, direction) diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 177c0e8a5..5645d2bcf 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -18,6 +18,7 @@ Dependencies: """ import re +from dataclasses import dataclass from typing import FrozenSet, List, Optional, Sequence, Tuple import nltk @@ -198,7 +199,24 @@ def parse_start_end_tags( return (None, current_tag_index) -def concatenate_aggregated_text(text_parts: List[str], add_spaces: bool) -> str: +@dataclass +class TextPartForConcatenation: + """Class representing a part of text for concatenation with concatenate_aggregated_text. + + Attributes: + text: The text content. + includes_inter_part_spaces: Whether any necessary inter-frame + (leading/trailing) spaces are already included in the text. + """ + + text: str + includes_inter_part_spaces: bool + + def __str__(self): + return f"{self.name}(text: [{self.text}], includes_inter_part_spaces: {self.includes_inter_part_spaces})" + + +def concatenate_aggregated_text(text_parts: List[TextPartForConcatenation]) -> str: """Concatenate a list of text parts into a single string. This function joins the provided list of text parts into a single string, @@ -208,15 +226,55 @@ def concatenate_aggregated_text(text_parts: List[str], add_spaces: bool) -> str: transcription services. Args: - text_parts: A list of strings representing parts of text to concatenate. - add_spaces: Whether to add spaces between text parts during concatenation. + text_parts: A list of text parts to concatenate. Returns: A single concatenated string. """ - # Concatenate text parts with or without spaces based on the flag - separator = " " if add_spaces else "" - result = separator.join(text_parts) + result = "" + last_includes_inter_part_spaces = False + + if not text_parts: + return result + + def append_part(part: TextPartForConcatenation): + nonlocal result + nonlocal last_includes_inter_part_spaces + result += part.text + last_includes_inter_part_spaces = part.includes_inter_part_spaces + + for part in text_parts: + # Part is empty. + # Skip. + if not part.text: + continue + + # Result is as yet empty. + # Just append. + if not result: + append_part(part) + continue + + if part.includes_inter_part_spaces and last_includes_inter_part_spaces: + # This part is part of an ongoing run that has spaces already included. + # Just append. + append_part(part) + elif not part.includes_inter_part_spaces and not last_includes_inter_part_spaces: + # This part is part of an ongoing run that has no spaces included. + # Add a space before appending. + result += " " + append_part(part) + else: + # This part represents a transition to a new run (spaces -> no spaces, or vice versa). + # Add a space if needed, before appending. + if not result[-1].isspace() and not part.text[0].isspace(): + result += " " + append_part(part) + + # NOTE: the above logic assumes that runs of text parts with + # includes_inter_part_spaces=True are well-formed, i.e. they're not + # actually multiple separate runs with a space-less boundary, like + # "hello ", "world.", "goodnight ", "moon." # Clean up any excessive whitespace result = result.strip() diff --git a/tests/test_context_aggregators.py b/tests/test_context_aggregators.py index 12be482c1..d43eaeaf3 100644 --- a/tests/test_context_aggregators.py +++ b/tests/test_context_aggregators.py @@ -1005,3 +1005,53 @@ class TestLLMAssistantAggregator( ) -> Optional[LLMAssistantAggregatorParams]: kwargs.pop("expect_stripped_words", None) return LLMAssistantAggregatorParams(**kwargs) if kwargs else None + + async def test_multiple_text_mixed(self): + assert self.CONTEXT_CLASS is not None, "CONTEXT_CLASS must be set in a subclass" + assert self.AGGREGATOR_CLASS is not None, "AGGREGATOR_CLASS must be set in a subclass" + + context = self.CONTEXT_CLASS() + aggregator = self.AGGREGATOR_CLASS( + context, params=self.create_assistant_aggregator_params(expect_stripped_words=False) + ) + + # The newer LLMAssistantAggregator expects TextFrames to declare + # when they include inter-frame spaces. + def make_text_frame(text: str, includes_spaces: bool) -> TextFrame: + frame = TextFrame(text=text) + frame.includes_inter_frame_spaces = includes_spaces + return frame + + frames_to_send = [ + LLMFullResponseStartFrame(), + make_text_frame("Hello ", includes_spaces=True), + make_text_frame("Pipecat. ", includes_spaces=True), + make_text_frame("Here's some", includes_spaces=True), + make_text_frame( + " code:", includes_spaces=True + ), # Validates ending includes_inter_frame_spaces run with no space + make_text_frame("```python\nprint('Hello, World!')\n```", includes_spaces=False), + make_text_frame( + "```javascript\nconsole.log('Hello, World!');\n```", includes_spaces=False + ), + make_text_frame( + " And some more: ", includes_spaces=True + ), # Validates starting includes_inter_frame_spaces run with a space and ending it with no space + make_text_frame("```html\n
Hello, World!
\n```", includes_spaces=False), + make_text_frame( + "Hope that ", includes_spaces=True + ), # Validates starting includes_inter_frame_spaces run with no space + make_text_frame("helps!", includes_spaces=True), + LLMFullResponseEndFrame(), + ] + expected_down_frames = [*self.EXPECTED_CONTEXT_FRAMES] + await run_test( + aggregator, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + self.check_message_content( + context, + 0, + "Hello Pipecat. Here's some code: ```python\nprint('Hello, World!')\n``` ```javascript\nconsole.log('Hello, World!');\n``` And some more: ```html\n
Hello, World!
\n``` Hope that helps!", + ) From 771469b834b25e1d679d5de80cceee2e77a2d23b Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 21:39:29 +0100 Subject: [PATCH 064/110] fix changelog --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b346e002d..3806c2062 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,10 +22,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT service from ElevenLabs. -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no - ONNX dependency or added processing complexity. - - Added word-level timestamps support to Hume TTS service ### Changed From 6484855139dc0d5265bd3210aff9dcc146717645 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 18 Nov 2025 21:47:46 +0100 Subject: [PATCH 065/110] fix changelog --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3806c2062..47be7d2bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,10 +86,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no - ONNX dependency or added processing complexity. - ## [0.0.94] - 2025-11-10 ### Changed From 1fcaf3a4bfd680ab270829ea1c738214b171e2f4 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 18 Nov 2025 21:38:49 -0300 Subject: [PATCH 066/110] Revert "Searching in both _function_calls_context_messages and context messages when updating the result." This reverts commit fccc91e923fc16617c45c1e001a086c99cfc6d78. --- .../processors/aggregators/llm_response_universal.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 8c084f2dc..93b1dcfaa 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -802,12 +802,7 @@ class LLMAssistantAggregator(LLMContextAggregator): del self._function_calls_in_progress[frame.tool_call_id] def _update_function_call_result(self, function_name: str, tool_call_id: str, result: Any): - def iter_all(): - yield from self._function_calls_context_messages - # In case on long-running function call, the function may already be added to the context - yield from self._context.get_messages() - - for message in iter_all(): + for message in self._function_calls_context_messages: if ( not isinstance(message, LLMSpecificMessage) and message["role"] == "tool" From 793aca6b8b1ae4f6450eb91b38552bc9254dbf61 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 18 Nov 2025 21:38:49 -0300 Subject: [PATCH 067/110] Revert "Ensure that the function call results respect the previous LLM context." This reverts commit a510b276e6ba1dbb678f4bd6b3b17fbdc61e409b. --- CHANGELOG.md | 5 -- .../aggregators/llm_response_universal.py | 74 +++++++------------ 2 files changed, 25 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10aa1c662..82da28b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,11 +71,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Fixed a race condition where, if the LLM received instructions to both produce - text and invoke a function call at the same time, the context would not be - updated before the function call result arrived, causing the bot to repeat - itself. - - Fixed an issue in the `Runner` where, when using `SmallWebRTCTransport`, the `request_data` was not being passed to the `SmallWebRTCRunnerArguments` body. diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 93b1dcfaa..d0ac67257 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -591,8 +591,6 @@ class LLMAssistantAggregator(LLMContextAggregator): self._started = 0 self._function_calls_in_progress: Dict[str, Optional[FunctionCallInProgressFrame]] = {} self._context_updated_tasks: Set[asyncio.Task] = set() - self._function_calls_context_messages = [] - self._function_calls_pending_context_updates_callbacks = [] @property def has_function_calls_in_progress(self) -> bool: @@ -649,23 +647,21 @@ class LLMAssistantAggregator(LLMContextAggregator): async def push_aggregation(self): """Push the current assistant aggregation with timestamp.""" - if self._aggregation: - aggregation = self.aggregation_string() - await self.reset() + if not self._aggregation: + return - if aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) + aggregation = self.aggregation_string() + await self.reset() - # Push context frame - await self.push_context_frame() + if aggregation: + self._context.add_message({"role": "assistant", "content": aggregation}) - # Push timestamp frame with current time - timestamp_frame = LLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) + # Push context frame + await self.push_context_frame() - if self._function_calls_context_messages: - self._flush_function_call_messages_to_context() - await self.push_context_frame(FrameDirection.UPSTREAM) + # Push timestamp frame with current time + timestamp_frame = LLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) async def _handle_llm_run(self, frame: LLMRunFrame): await self.push_context_frame(FrameDirection.UPSTREAM) @@ -685,23 +681,6 @@ class LLMAssistantAggregator(LLMContextAggregator): self._started = 0 await self.reset() - def _flush_function_call_messages_to_context(self): - """Move all function calls messages into context, then clear the list.""" - if self._function_calls_context_messages: - self._context.add_messages(self._function_calls_context_messages) - self._function_calls_context_messages.clear() - - # Call the `on_context_updated` callbacks once the function call results - # are added to the context. Run them in separate tasks to make - # sure we don't block the pipeline. - for callback, task_name in self._function_calls_pending_context_updates_callbacks: - task = self.create_task(callback(), task_name) - self._context_updated_tasks.add(task) - task.add_done_callback(self._context_updated_task_finished) - - # Clear the pending callbacks list - self._function_calls_pending_context_updates_callbacks.clear() - async def _handle_function_calls_started(self, frame: FunctionCallsStartedFrame): function_names = [f"{f.function_name}:{f.tool_call_id}" for f in frame.function_calls] logger.debug(f"{self} FunctionCallsStartedFrame: {function_names}") @@ -714,7 +693,7 @@ class LLMAssistantAggregator(LLMContextAggregator): ) # Update context with the in-progress function call - self._function_calls_context_messages.append( + self._context.add_message( { "role": "assistant", "tool_calls": [ @@ -729,7 +708,7 @@ class LLMAssistantAggregator(LLMContextAggregator): ], } ) - self._function_calls_context_messages.append( + self._context.add_message( { "role": "tool", "content": "IN_PROGRESS", @@ -760,13 +739,6 @@ class LLMAssistantAggregator(LLMContextAggregator): else: self._update_function_call_result(frame.function_name, frame.tool_call_id, "COMPLETED") - # Store the on_context_updated callback along with task name info to be invoked later - if properties and properties.on_context_updated: - task_name = f"{frame.function_name}:{frame.tool_call_id}:on_context_updated" - self._function_calls_pending_context_updates_callbacks.append( - (properties.on_context_updated, task_name) - ) - run_llm = False # Run inference if the function call result requires it. @@ -781,13 +753,17 @@ class LLMAssistantAggregator(LLMContextAggregator): # If this is the last function call in progress, run the LLM. run_llm = not bool(self._function_calls_in_progress) - # Only run if the LLM response has completed (not currently generating), - # otherwise defer execution until push_aggregation() is called - # (triggered by LLMFullResponseEndFrame or interruption). - if not self._started: - self._flush_function_call_messages_to_context() - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) + if run_llm: + await self.push_context_frame(FrameDirection.UPSTREAM) + + # Call the `on_context_updated` callback once the function call result + # is added to the context. Also, run this in a separate task to make + # sure we don't block the pipeline. + if properties and properties.on_context_updated: + task_name = f"{frame.function_name}:{frame.tool_call_id}:on_context_updated" + task = self.create_task(properties.on_context_updated(), task_name) + self._context_updated_tasks.add(task) + task.add_done_callback(self._context_updated_task_finished) async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame): logger.debug( @@ -802,7 +778,7 @@ class LLMAssistantAggregator(LLMContextAggregator): del self._function_calls_in_progress[frame.tool_call_id] def _update_function_call_result(self, function_name: str, tool_call_id: str, result: Any): - for message in self._function_calls_context_messages: + for message in self._context.get_messages(): if ( not isinstance(message, LLMSpecificMessage) and message["role"] == "tool" From ceaf53fdb0cd9f9f1fe7c58c2ae62e454b8915fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Nov 2025 12:08:09 -0800 Subject: [PATCH 068/110] LLMContext: async create_image_message/create_audio_message fixes --- CHANGELOG.md | 7 ++++--- .../22d-natural-conversation-gemini-audio.py | 2 +- src/pipecat/processors/aggregators/llm_context.py | 14 ++++++++------ .../aggregators/llm_response_universal.py | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47be7d2bf..5ace3aa14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,9 +26,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- ⚠️ Breaking change: `LLMContext.create_image_message()` and - `LLMContext.create_audio_message()` are now async methods. This fixes and - issue where the asyncio event loop would be blocked while encoding audio or +- ⚠️ Breaking change: `LLMContext.create_image_message()`, + `LLMContext.create_audio_message()`, `LLMContext.add_image_frame_message()` + and `LLMContext.add_audio_frames_message()` are now async methods. This fixes + an issue where the asyncio event loop would be blocked while encoding audio or images. - `ConsumerProcessor` now queues frames from the producer internally instead of diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 7a7155297..a7837ce60 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -391,7 +391,7 @@ class AudioAccumulator(FrameProcessor): ) self._user_speaking = False context = LLMContext() - context.add_audio_frames_message(audio_frames=self._audio_frames) + await context.add_audio_frames_message(audio_frames=self._audio_frames) await self.push_frame(LLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index b9216103a..99b9aeaa9 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -180,7 +180,7 @@ class LLMContext: text: Optional text to include with the audio. """ - def encode_audio(): + async def encode_audio(): sample_rate = audio_frames[0].sample_rate num_channels = audio_frames[0].num_channels @@ -198,7 +198,7 @@ class LLMContext: encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") return encoded_audio - encoded_audio = asyncio.to_thread(encode_audio) + encoded_audio = await asyncio.to_thread(encode_audio) content.append( { @@ -333,7 +333,7 @@ class LLMContext: """ self._tool_choice = tool_choice - def add_image_frame_message( + async def add_image_frame_message( self, *, format: str, size: tuple[int, int], image: bytes, text: Optional[str] = None ): """Add a message containing an image frame. @@ -344,10 +344,12 @@ class LLMContext: image: Raw image bytes. text: Optional text to include with the image. """ - message = LLMContext.create_image_message(format=format, size=size, image=image, text=text) + message = await LLMContext.create_image_message( + format=format, size=size, image=image, text=text + ) self.add_message(message) - def add_audio_frames_message( + async def add_audio_frames_message( self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows" ): """Add a message containing audio frames. @@ -356,7 +358,7 @@ class LLMContext: audio_frames: List of audio frame objects to include. text: Optional text to include with the audio. """ - message = LLMContext.create_audio_message(audio_frames=audio_frames, text=text) + message = await LLMContext.create_audio_message(audio_frames=audio_frames, text=text) self.add_message(message) @staticmethod diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index d0ac67257..87974bed2 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -793,7 +793,7 @@ class LLMAssistantAggregator(LLMContextAggregator): logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})") - self._context.add_image_frame_message( + await self._context.add_image_frame_message( format=frame.format, size=frame.size, image=frame.image, From 39b4e6183765b8d59d34d0d93900d4b7057e9b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Nov 2025 13:50:47 -0800 Subject: [PATCH 069/110] SimliVideoService: fix connection issue --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- src/pipecat/services/simli/video.py | 16 +++++++++++++--- uv.lock | 9 +++++---- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ace3aa14..ff3934b2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `SimliVideoService` connection issue. + - Fixed an issue in the `Runner` where, when using `SmallWebRTCTransport`, the `request_data` was not being passed to the `SmallWebRTCRunnerArguments` body. diff --git a/pyproject.toml b/pyproject.toml index e313c789d..9354c2066 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,7 +99,7 @@ local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "tor local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] remote-smart-turn = [] silero = [ "onnxruntime>=1.20.1,<2" ] -simli = [ "simli-ai~=0.1.25"] +simli = [ "simli-ai~=1.0.3"] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.1" ] speechmatics = [ "speechmatics-rt>=0.5.0" ] diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index bac54f35b..e6514ca7b 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -84,6 +84,10 @@ class SimliVideoService(FrameProcessor): Please use 'api_key' and 'face_id' parameters instead. use_turn_server: Whether to use TURN server for connection. Defaults to False. + + .. deprecated:: 0.0.95 + The 'use_turn_server' parameter is deprecated and will be removed in a future version. + latency_interval: Latency interval setting for sending health checks to check the latency to Simli Servers. Defaults to 0. simli_url: URL of the simli servers. Can be changed for custom deployments @@ -135,14 +139,20 @@ class SimliVideoService(FrameProcessor): config = SimliConfig(**config_kwargs) + if use_turn_server: + warnings.warn( + "The 'use_turn_server' parameter is deprecated and will be removed in a future version.", + DeprecationWarning, + stacklevel=2, + ) + self._initialized = False # Add buffer time to session limits config.maxIdleTime += 5 config.maxSessionLength += 5 self._simli_client = SimliClient( - config, - use_turn_server, - latency_interval, + config=config, + latencyInterval=latency_interval, simliURL=simli_url, ) diff --git a/uv.lock b/uv.lock index eabe03714..6dec2d8dc 100644 --- a/uv.lock +++ b/uv.lock @@ -4727,7 +4727,7 @@ requires-dist = [ { name = "resampy", specifier = "~=0.4.3" }, { name = "sarvamai", marker = "extra == 'sarvam'", specifier = "==0.1.21" }, { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.28.0,<3" }, - { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=0.1.25" }, + { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=1.0.3" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, { name = "soxr", specifier = "~=0.5.0" }, { name = "speechmatics-rt", marker = "extra == 'speechmatics'", specifier = ">=0.5.0" }, @@ -6496,18 +6496,19 @@ wheels = [ [[package]] name = "simli-ai" -version = "0.1.25" +version = "1.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiortc" }, { name = "av" }, { name = "httpx" }, + { name = "livekit" }, { name = "numpy" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/64/6a/b28f90baf76f6a60865985f6233ff44abc72d45b66b76658bff3961e20a7/simli_ai-0.1.25.tar.gz", hash = "sha256:7a00b3426dc26a6a421641072c3e49014b7950c621cf4544152f35c58d13fcff", size = 13182, upload-time = "2025-11-06T16:27:08.862Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/03/b0b3e12c68fd3f9c57f6afeee67841349e4866b88760f413357af3043ae4/simli_ai-1.0.3.tar.gz", hash = "sha256:e96b0621a1dbd9582b2ae3d51eefd4995983b49c1f1061eb9239707b15a1ee27", size = 13350, upload-time = "2025-11-13T12:22:32.514Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/57/ae1032fd88214ea4ee6d3028c817c12a999eb90a67766bbab31e9819385a/simli_ai-0.1.25-py3-none-any.whl", hash = "sha256:7d01f65321dc9052f25e15d0463af6a20a86c6d37d9a7b3a2c4b01cbec0a54ed", size = 13651, upload-time = "2025-11-06T16:27:07.765Z" }, + { url = "https://files.pythonhosted.org/packages/5e/d1/dc382ba529de0d2d51f35e9bfd20b41d8f5c96404a3aa24bae97a5a5e51f/simli_ai-1.0.3-py3-none-any.whl", hash = "sha256:ffafa7540aa28833e207be8f3b199367c7f500dac1a8ba0108395bfb7d8362bc", size = 13863, upload-time = "2025-11-13T12:22:31.218Z" }, ] [[package]] From 51ba245e1030af0f7b7366f5d804fff3d6165c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Nov 2025 14:13:34 -0800 Subject: [PATCH 070/110] scripts(evals): fix EVAL_CONVERSATION/EVAL_WEATHER eval --- scripts/evals/run-release-evals.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index da6df053d..4f8268d78 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -30,8 +30,8 @@ EVAL_SIMPLE_MATH = EvalConfig( ) EVAL_WEATHER = EvalConfig( - prompt="What's the weather in San Francisco?", - eval="The user says something specific about the current weather in San Francisco, including the degrees.", + prompt="What's the weather in San Francisco (in farhenheit or celsius)?", + eval="The user says something specific about the current weather in San Francisco, including the degrees (in farhenheit or celsius).", ) EVAL_ONLINE_SEARCH = EvalConfig( @@ -70,7 +70,7 @@ EVAL_VOICEMAIL = EvalConfig( EVAL_CONVERSATION = EvalConfig( prompt="Hello, this is Mark.", - eval="The user replies with a greeting.", + eval="The user acknowledges the greeting.", eval_speaks_first=True, ) From cf1a9c1548d46b616a071bedcf3991ee8bd04ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Nov 2025 10:54:44 -0800 Subject: [PATCH 071/110] update CHANGELOG for 0.0.95 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff3934b2e..d69eb897a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.95] - 2025-11-18 ### Added From 42423bff41892d28592ac2cd8f80a306fa70213e Mon Sep 17 00:00:00 2001 From: vipyne Date: Tue, 18 Nov 2025 19:29:32 -0600 Subject: [PATCH 072/110] update MCP foundational examples --- examples/foundational/39-mcp-stdio.py | 2 +- examples/foundational/39a-mcp-run-sse.py | 159 ---------------------- examples/foundational/39b-multiple-mcp.py | 52 ++++--- 3 files changed, 33 insertions(+), 180 deletions(-) delete mode 100644 examples/foundational/39a-mcp-run-sse.py diff --git a/examples/foundational/39-mcp-stdio.py b/examples/foundational/39-mcp-stdio.py index b88ee30b1..7127d831f 100644 --- a/examples/foundational/39-mcp-stdio.py +++ b/examples/foundational/39-mcp-stdio.py @@ -155,7 +155,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. You have access to tools to search the Rijksmuseum collection. - Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. + Offer, for example, to show a floral still life, use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. diff --git a/examples/foundational/39a-mcp-run-sse.py b/examples/foundational/39a-mcp-run-sse.py deleted file mode 100644 index 328af6ce4..000000000 --- a/examples/foundational/39a-mcp-run-sse.py +++ /dev/null @@ -1,159 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - - -import os - -from dotenv import load_dotenv -from loguru import logger -from mcp.client.session_group import SseServerParameters - -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.anthropic.llm import AnthropicLLMService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.mcp_service import MCPClient -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams - -load_dotenv(override=True) - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-7-sonnet-latest" - ) - - try: - # https://docs.mcp.run/integrating/tutorials/mcp-run-sse-openai-agents/ - mcp = MCPClient(server_params=SseServerParameters(url=os.getenv("MCP_RUN_SSE_URL"))) - except Exception as e: - logger.error(f"error setting up mcp") - logger.exception("error trace:") - - tools = {} - try: - tools = await mcp.register_tools(llm) - except Exception as e: - logger.error(f"error registering tools") - logger.exception("error trace:") - - system = f""" - You are a helpful LLM in a WebRTC call. - Your goal is to demonstrate your capabilities in a succinct way. - You have access to a number of tools provided by mcp.run. Use any and all tools to help users. - Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. - Respond to what the user said in a creative and helpful way. - When asked for today's date, use 'https://www.datetoday.net/'. - Don't overexplain what you are doing. - Just respond with short sentences when you are carrying out tool calls. - """ - - messages = [{"role": "system", "content": system}] - - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, - context_aggregator.user(), # User spoken responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses and tool context - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - if not os.getenv("MCP_RUN_SSE_URL"): - logger.error( - f"Please set MCP_RUN_SSE_URL environment variable for this example. See https://mcp.run" - ) - import sys - - sys.exit(1) - - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/39b-multiple-mcp.py b/examples/foundational/39b-multiple-mcp.py index dad059208..88fdc3610 100644 --- a/examples/foundational/39b-multiple-mcp.py +++ b/examples/foundational/39b-multiple-mcp.py @@ -7,6 +7,7 @@ import asyncio import io +import json import os import re import shutil @@ -15,7 +16,7 @@ import aiohttp from dotenv import load_dotenv from loguru import logger from mcp import StdioServerParameters -from mcp.client.session_group import SseServerParameters +from mcp.client.session_group import StreamableHttpParameters from PIL import Image from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -66,10 +67,12 @@ class UrlToImageProcessor(FrameProcessor): await self.push_frame(frame, direction) def extract_url(self, text: str): - pattern = r"!\[[^\]]*\]\((https?://[^)]+\.(png|jpg|jpeg|PNG|JPG|JPEG|gif))\)" - match = re.search(pattern, text) - if match: - return match.group(1) + data = json.loads(text) + if "artObject" in data: + return data["artObject"]["webImage"]["url"] + if "artworks" in data and len(data["artworks"]): + return data["artworks"][0]["webImage"]["url"] + return None async def run_image_process(self, image_url: str): @@ -132,10 +135,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): system = f""" You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. - You have access to tools to search the Rijksmuseum collection. - Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. + You have access to tools to search the Rijksmuseum collection and the user's GitHub repositories and account. + Offer, for example, to show a floral still life, use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. + You can also offer to answer users questions about their GitHub repositories and account. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. Don't overexplain what you are doing. @@ -145,11 +149,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "system", "content": system}] try: - mcp = MCPClient( + rijksmuseum_mcp = MCPClient( server_params=StdioServerParameters( command=shutil.which("npx"), # https://github.com/r-huijts/rijksmuseum-mcp - args=["-y", "mcp-server-error setting up mcp"], + args=["-y", "mcp-server-rijksmuseum"], env={"RIJKSMUSEUM_API_KEY": os.getenv("RIJKSMUSEUM_API_KEY")}, ) ) @@ -157,24 +161,32 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.error(f"error setting up rijksmuseum mcp") logger.exception("error trace:") try: - # https://docs.mcp.run/integrating/tutorials/mcp-run-sse-openai-agents/ - # ie. "https://www.mcp.run/api/mcp/sse?..." - # ensure the profile has a tool or few installed - mcp_run = MCPClient(server_params=SseServerParameters(url=os.getenv("MCP_RUN_SSE_URL"))) + # Github MCP docs: https://github.com/github/github-mcp-server + # Enable Github Copilot on your GitHub account. Free tier is ok. (https://github.com/settings/copilot) + # Generate a personal access token. It must be a Fine-grained token, classic tokens are not supported. (https://github.com/settings/personal-access-tokens) + # Set permissions you want to use (eg. "all repositories", "profile: read/write", etc) + github_mcp = MCPClient( + server_params=StreamableHttpParameters( + url="https://api.githubcopilot.com/mcp/", + headers={ + "Authorization": f"Bearer {os.getenv('GITHUB_PERSONAL_ACCESS_TOKEN')}" + }, + ) + ) except Exception as e: logger.error(f"error setting up mcp.run") logger.exception("error trace:") - tools = {} - run_tools = {} + rijksmuseum_tools = {} + github_tools = {} try: - tools = await mcp.register_tools(llm) - run_tools = await mcp_run.register_tools(llm) + rijksmuseum_tools = await rijksmuseum_mcp.register_tools(llm) + github_tools = await github_mcp.register_tools(llm) except Exception as e: logger.error(f"error registering tools") logger.exception("error trace:") - all_standard_tools = run_tools.standard_tools + tools.standard_tools + all_standard_tools = rijksmuseum_tools.standard_tools + github_tools.standard_tools all_tools = ToolsSchema(standard_tools=all_standard_tools) context = LLMContext(messages, all_tools) @@ -226,9 +238,9 @@ async def bot(runner_args: RunnerArguments): if __name__ == "__main__": - if not os.getenv("RIJKSMUSEUM_API_KEY") or not os.getenv("MCP_RUN_SSE_URL"): + if not os.getenv("RIJKSMUSEUM_API_KEY") or not os.getenv("GITHUB_PERSONAL_ACCESS_TOKEN"): logger.error( - f"Please set RIJKSMUSEUM_API_KEY and MCP_RUN_SSE_URL environment variables. See https://github.com/r-huijts/rijksmuseum-mcp and https://mcp.run" + f"Please set `RIJKSMUSEUM_API_KEY` and `GITHUB_PERSONAL_ACCESS_TOKEN` environment variables. See https://github.com/r-huijts/rijksmuseum-mcp." ) import sys From 68292bd75fb0b48f74c6f82ffcdb68b47424258a Mon Sep 17 00:00:00 2001 From: vipyne Date: Wed, 19 Nov 2025 10:34:13 -0600 Subject: [PATCH 073/110] rename MCP foundational examples --- .../{39c-mcp-run-http.py => 39a-mcp-streamable-http.py} | 0 ...http-gemini-live.py => 39b-mcp-streamable-http-gemini-live.py} | 0 .../foundational/{39b-multiple-mcp.py => 39c-multiple-mcp.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename examples/foundational/{39c-mcp-run-http.py => 39a-mcp-streamable-http.py} (100%) rename examples/foundational/{39d-mcp-run-http-gemini-live.py => 39b-mcp-streamable-http-gemini-live.py} (100%) rename examples/foundational/{39b-multiple-mcp.py => 39c-multiple-mcp.py} (100%) diff --git a/examples/foundational/39c-mcp-run-http.py b/examples/foundational/39a-mcp-streamable-http.py similarity index 100% rename from examples/foundational/39c-mcp-run-http.py rename to examples/foundational/39a-mcp-streamable-http.py diff --git a/examples/foundational/39d-mcp-run-http-gemini-live.py b/examples/foundational/39b-mcp-streamable-http-gemini-live.py similarity index 100% rename from examples/foundational/39d-mcp-run-http-gemini-live.py rename to examples/foundational/39b-mcp-streamable-http-gemini-live.py diff --git a/examples/foundational/39b-multiple-mcp.py b/examples/foundational/39c-multiple-mcp.py similarity index 100% rename from examples/foundational/39b-multiple-mcp.py rename to examples/foundational/39c-multiple-mcp.py From 30ff488714dac90743224ddf948bb2b8d33fa05b Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Wed, 19 Nov 2025 17:04:07 -0500 Subject: [PATCH 074/110] Fix typo in event handler documentation --- src/pipecat/services/stt_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index 6fb96c571..7c8bae6bf 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -38,7 +38,7 @@ class STTService(AIService): Event handlers: on_connected: Called when connected to the STT service. - on_connected: Called when disconnected from the STT service. + on_disconnected: Called when disconnected from the STT service. on_connection_error: Called when a connection to the STT service error occurs. Example:: From 1cc69d475da52336161de2cc9c88318026de3008 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Wed, 19 Nov 2025 22:49:16 -0500 Subject: [PATCH 075/110] feat: Add speaking rate control to Inworld TTS service & fix param cases --- src/pipecat/services/inworld/tts.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index dc2282b91..ab218b3c0 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -146,6 +146,8 @@ class InworldTTSService(TTSService): Parameters: temperature: Voice temperature control for synthesis variability (e.g., 1.1). Valid range: [0, 2]. Higher values increase variability. + speaking_rate: Speaking speed control (range: [0.5, 1.5]). Defaults to 1.0 when + unset. Note: Language is automatically inferred from the input text by Inworld's TTS models, @@ -153,6 +155,7 @@ class InworldTTSService(TTSService): """ temperature: Optional[float] = None # optional temperature control (range: [0, 2]) + speaking_rate: Optional[float] = None # optional speaking rate control (range: [0.5, 1.5]) def __init__( self, @@ -198,6 +201,7 @@ class InworldTTSService(TTSService): - Other formats as supported by Inworld API params: Optional input parameters for additional configuration. Use this to specify: - temperature: Voice temperature control for variability (range: [0, 2], e.g., 1.1, optional) + - speaking_rate: Set desired speaking speed (range: [0.5, 1.5], optional) Language is automatically inferred from input text. **kwargs: Additional arguments passed to the parent TTSService class. @@ -228,15 +232,18 @@ class InworldTTSService(TTSService): self._settings = { "voiceId": voice_id, # Voice selection from direct parameter "modelId": model, # TTS model selection from direct parameter - "audio_config": { # Audio format configuration - "audio_encoding": encoding, # Format: LINEAR16, MP3, etc. - "sample_rate_hertz": 0, # Will be set in start() from parent service + "audioConfig": { # Audio format configuration + "audioEncoding": encoding, # Format: LINEAR16, MP3, etc. + "sampleRateHertz": 0, # Will be set in start() from parent service }, } # Add optional temperature parameter if provided (valid range: [0, 2]) if params and params.temperature is not None: self._settings["temperature"] = params.temperature + # Add optional speaking rate if provided (valid range: [0.5, 1.5]) + if params and params.speaking_rate is not None: + self._settings["audioConfig"]["speakingRate"] = params.speaking_rate # Register voice and model with parent service for metrics and tracking self.set_voice(voice_id) # Used for logging and metrics @@ -257,7 +264,7 @@ class InworldTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["audio_config"]["sample_rate_hertz"] = self.sample_rate + self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate async def stop(self, frame: EndFrame): """Stop the Inworld TTS service. @@ -323,9 +330,7 @@ class InworldTTSService(TTSService): "text": text, # Text to synthesize "voiceId": self._settings["voiceId"], # Voice selection (Ashley, Hades, etc.) "modelId": self._settings["modelId"], # TTS model (inworld-tts-1) - "audio_config": self._settings[ - "audio_config" - ], # Audio format settings (LINEAR16, 48kHz) + "audioConfig": self._settings["audioConfig"], # Audio format settings (LINEAR16, 48kHz) } # Add optional temperature parameter if configured (valid range: [0, 2]) From fa6b8851ed7477d12d90d6cca72d74a551acae9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Nov 2025 21:56:38 -0800 Subject: [PATCH 076/110] pyproject: update daily-python to 0.22.0 --- CHANGELOG.md | 6 ++++++ pyproject.toml | 2 +- uv.lock | 12 ++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d69eb897a..d85e6b2b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- Updated `daily-python` to 0.22.0. + ## [0.0.95] - 2025-11-18 ### Added diff --git a/pyproject.toml b/pyproject.toml index 9354c2066..73da9083a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] cerebras = [] deepseek = [] -daily = [ "daily-python~=0.21.0" ] +daily = [ "daily-python~=0.22.0" ] deepgram = [ "deepgram-sdk~=4.7.0" ] elevenlabs = [ "pipecat-ai[websockets-base]" ] fal = [ "fal-client~=0.5.9" ] diff --git a/uv.lock b/uv.lock index 6dec2d8dc..662150b5b 100644 --- a/uv.lock +++ b/uv.lock @@ -1291,13 +1291,13 @@ wheels = [ [[package]] name = "daily-python" -version = "0.21.0" +version = "0.22.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/11/99590f8b7aad077f3f9b5b59d39b010aee0bd01b14dece8ae1e93d8080e7/daily_python-0.21.0-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:bdec96417825181559769bb2258ae688d1215949a1878336194e36fb452274a8", size = 13277066, upload-time = "2025-10-29T00:20:49.523Z" }, - { url = "https://files.pythonhosted.org/packages/e5/db/8c57f1a1b713ba3393584ac2be32d8074d3022a2c2c17c28eb4cd2aa3629/daily_python-0.21.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:18677fa1415a0dc48b891cdf2fb8fe9dabc70e1b019d5aaa3d0699ccc8d187c9", size = 11644908, upload-time = "2025-10-29T00:20:52.106Z" }, - { url = "https://files.pythonhosted.org/packages/64/b6/b03f2f58a367d6ef4bb728715471542fdfa68afa8a177670139c3a2aadb7/daily_python-0.21.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:97eb97352fe74227061b678e330b8befcfa4c694feb6eb2b09fe6eacec00ad6d", size = 13652356, upload-time = "2025-10-29T00:20:54.813Z" }, - { url = "https://files.pythonhosted.org/packages/f6/76/bde65f6f8d4c1679dc6c185fa37dae9223f6ddb4b7ced728ef46504956f7/daily_python-0.21.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:68c3e36f609fc2fce79e4d17ecf1021eadd836506db6c5125f95c682bcf3612a", size = 14304643, upload-time = "2025-10-29T00:20:57.194Z" }, + { url = "https://files.pythonhosted.org/packages/04/db/d6c311ba760123a9987a9ae291171c9a6af11ee4dbefdb661d65e2ac13a2/daily_python-0.22.0-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:2ef7591a7929c5d9e5ea78329ea049d2f313bd3d2d289f5f4ecce4bb3799c3d0", size = 13526264, upload-time = "2025-11-20T05:52:04.134Z" }, + { url = "https://files.pythonhosted.org/packages/ab/47/f1f6d893e7aab4b2e3d3b20d0dd8fbf31c7a71597d274aae1d288e36fac3/daily_python-0.22.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:0fbc8467c471ce536dc6214a24cc28cbc38ff61113b1714e09d0eafc2741fc5a", size = 12041400, upload-time = "2025-11-20T05:52:06.419Z" }, + { url = "https://files.pythonhosted.org/packages/9d/c9/8f26944cd55ece2ab9c076fae5c1fcf4fdc8639ea6f2b861566d26ad9e00/daily_python-0.22.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b8aa9531b0fa2852b41935697d184be86318eaee3b35f49e0b5714e53cdb524a", size = 14147194, upload-time = "2025-11-20T05:52:08.4Z" }, + { url = "https://files.pythonhosted.org/packages/a8/02/d86f4cee39bcdb112d83e2cf12345d7a974cbad5dafb350788148644f16b/daily_python-0.22.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8f49fceee92830aaf53a65053f332504b2cc62af79c38edb9bce8707c9fa4b0c", size = 14654605, upload-time = "2025-11-20T05:52:10.639Z" }, ] [[package]] @@ -4658,7 +4658,7 @@ requires-dist = [ { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.42.0" }, { name = "cartesia", marker = "extra == 'cartesia'", specifier = "~=2.0.3" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" }, - { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.21.0" }, + { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.22.0" }, { name = "deepgram-sdk", marker = "extra == 'deepgram'", specifier = "~=4.7.0" }, { name = "docstring-parser", specifier = "~=0.16" }, { name = "einops", marker = "extra == 'moondream'", specifier = "~=0.8.0" }, From ead361f665f6e610a9a2f71af3a202837fc7ece8 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Thu, 20 Nov 2025 07:45:13 -0500 Subject: [PATCH 077/110] fix --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d69eb897a..5749e60da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added word-level timestamps support to Hume TTS service +- Added optional speaking rate control to `InworldTTSService`. + ### Changed - ⚠️ Breaking change: `LLMContext.create_image_message()`, @@ -89,6 +91,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. +- Fixed `InworldTTSService` audio config payload to use camelCase keys expected + by the Inworld API. + ## [0.0.94] - 2025-11-10 ### Changed From c89f230c996c56bfc0ef9cbd3807bd2ca769b3fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Nov 2025 08:40:30 -0800 Subject: [PATCH 078/110] fix CHANGELOG --- CHANGELOG.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e081d2495..9887968dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added optional speaking rate control to `InworldTTSService`. + ### Changed - Updated `daily-python` to 0.22.0. +### Fixed + +- Fixed `InworldTTSService` audio config payload to use camelCase keys expected + by the Inworld API. + ## [0.0.95] - 2025-11-18 ### Added @@ -30,8 +39,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added word-level timestamps support to Hume TTS service -- Added optional speaking rate control to `InworldTTSService`. - ### Changed - ⚠️ Breaking change: `LLMContext.create_image_message()`, @@ -97,9 +104,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. -- Fixed `InworldTTSService` audio config payload to use camelCase keys expected - by the Inworld API. - ## [0.0.94] - 2025-11-10 ### Changed From f3cb5e01065559c9428df4712020d5c863deb873 Mon Sep 17 00:00:00 2001 From: minimax Date: Tue, 4 Nov 2025 02:04:25 +0800 Subject: [PATCH 079/110] feat(minimax): comprehensive updates to TTS service - Add support for speech-2.6-hd and speech-2.6-turbo models - Add 16 new languages (total 40): Afrikaans, Bulgarian, Catalan, Danish, Persian, Filipino, Hebrew, Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, Tamil - Add new emotions: calm and fluent - Add new parameters: text_normalization (renamed from english_normalization), latex_read, force_cbr, exclude_aggregated_audio, subtitle_enable, subtitle_type - Extract trace_id from response headers for all requests - Improve error handling for non-streaming error responses - Add detailed extra_info logging (audio_length, audio_size, usage_characters, word_count) - Add validation warnings for language/model compatibility - Fix silent error issue where HTTP 200 responses with errors were ignored BREAKING CHANGE: Renamed parameter english_normalization to text_normalization --- src/pipecat/__init__.py | 7 +- src/pipecat/services/minimax/tts.py | 269 +++++++++++++++++++++++++--- 2 files changed, 252 insertions(+), 24 deletions(-) diff --git a/src/pipecat/__init__.py b/src/pipecat/__init__.py index 1975e1654..ed9892e9d 100644 --- a/src/pipecat/__init__.py +++ b/src/pipecat/__init__.py @@ -5,11 +5,14 @@ # import sys -from importlib.metadata import version +from importlib.metadata import version, PackageNotFoundError from loguru import logger -__version__ = version("pipecat-ai") +try: + __version__ = version("pipecat-ai") +except PackageNotFoundError: + __version__ = "0.0.0.dev" # Development version logger.info(f"ᓚᘏᗢ Pipecat {__version__} (Python {sys.version}) ᓚᘏᗢ") diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 05e2dac3b..8b7f71179 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -40,24 +40,40 @@ def language_to_minimax_language(language: Language) -> Optional[str]: The corresponding MiniMax language name, or None if not supported. """ LANGUAGE_MAP = { + Language.AF: "Afrikaans", Language.AR: "Arabic", + Language.BG: "Bulgarian", + Language.CA: "Catalan", Language.CS: "Czech", + Language.DA: "Danish", Language.DE: "German", Language.EL: "Greek", Language.EN: "English", Language.ES: "Spanish", + Language.FA: "Persian", # ⚠️ Only supported by speech-2.6-* models Language.FI: "Finnish", + Language.FIL: "Filipino", # ⚠️ Only supported by speech-2.6-* models Language.FR: "French", + Language.HE: "Hebrew", Language.HI: "Hindi", + Language.HR: "Croatian", + Language.HU: "Hungarian", Language.ID: "Indonesian", Language.IT: "Italian", Language.JA: "Japanese", Language.KO: "Korean", + Language.MS: "Malay", + Language.NB: "Norwegian", + Language.NN: "Nynorsk", Language.NL: "Dutch", Language.PL: "Polish", Language.PT: "Portuguese", Language.RO: "Romanian", Language.RU: "Russian", + Language.SK: "Slovak", + Language.SL: "Slovenian", + Language.SV: "Swedish", + Language.TA: "Tamil", # ⚠️ Only supported by speech-2.6-* models Language.TH: "Thai", Language.TR: "Turkish", Language.UK: "Ukrainian", @@ -65,6 +81,9 @@ def language_to_minimax_language(language: Language) -> Optional[str]: Language.YUE: "Chinese,Yue", Language.ZH: "Chinese", } + + # Languages that require speech-2.6-* models + V26_ONLY_LANGUAGES = {Language.FA, Language.FIL, Language.TA} return resolve_language(language, LANGUAGE_MAP, use_base_code=False) @@ -84,13 +103,20 @@ class MiniMaxHttpTTSService(TTSService): """Configuration parameters for MiniMax TTS. Parameters: - language: Language for TTS generation. + language: Language for TTS generation. Supports 40 languages. + Note: Filipino, Tamil, and Persian require speech-2.6-* models. speed: Speech speed (range: 0.5 to 2.0). volume: Speech volume (range: 0 to 10). pitch: Pitch adjustment (range: -12 to 12). emotion: Emotional tone (options: "happy", "sad", "angry", "fearful", - "disgusted", "surprised", "neutral"). - english_normalization: Whether to apply English text normalization. + "disgusted", "surprised", "calm", "fluent"). + text_normalization: Enable text normalization (Chinese/English). + latex_read: Enable LaTeX formula reading. + force_cbr: Enable Constant Bitrate (CBR) for audio encoding (MP3 only). + exclude_aggregated_audio: Whether to exclude aggregated audio in final chunk. + subtitle_enable: Enable subtitle generation (non-streaming only). + subtitle_type: Subtitle timestamp granularity (options: "word", "sentence"). + Only effective when subtitle_enable is True. Defaults to "sentence". """ language: Optional[Language] = Language.EN @@ -98,7 +124,12 @@ class MiniMaxHttpTTSService(TTSService): volume: Optional[float] = 1.0 pitch: Optional[int] = 0 emotion: Optional[str] = None - english_normalization: Optional[bool] = None + text_normalization: Optional[bool] = None + latex_read: Optional[bool] = None + force_cbr: Optional[bool] = None + exclude_aggregated_audio: Optional[bool] = None + subtitle_enable: Optional[bool] = None + subtitle_type: Optional[str] = "sentence" def __init__( self, @@ -121,8 +152,10 @@ class MiniMaxHttpTTSService(TTSService): Global: https://api.minimax.io/v1/t2a_v2 Mainland China: https://api.minimaxi.chat/v1/t2a_v2 group_id: MiniMax Group ID to identify project. - model: TTS model name. Defaults to "speech-02-turbo". Options include - "speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo". + model: TTS model name. Defaults to "speech-02-turbo". Options include: + "speech-2.6-hd", "speech-2.6-turbo" (latest, supports Filipino/Tamil/Persian), + "speech-02-hd", "speech-02-turbo", + "speech-01-hd", "speech-01-turbo". voice_id: Voice identifier. Defaults to "Calm_Woman". aiohttp_session: aiohttp.ClientSession for API communication. sample_rate: Output audio sample rate in Hz. If None, uses pipeline default. @@ -139,6 +172,7 @@ class MiniMaxHttpTTSService(TTSService): self._session = aiohttp_session self._model_name = model self._voice_id = voice_id + self._current_trace_id: Optional[str] = None # Create voice settings self._settings = { @@ -155,6 +189,12 @@ class MiniMaxHttpTTSService(TTSService): }, } + # Add stream_options if exclude_aggregated_audio is set + if params.exclude_aggregated_audio is not None: + self._settings["stream_options"] = { + "exclude_aggregated_audio": params.exclude_aggregated_audio + } + # Set voice and model self.set_voice(voice_id) self.set_model_name(model) @@ -164,10 +204,21 @@ class MiniMaxHttpTTSService(TTSService): service_lang = self.language_to_service_language(params.language) if service_lang: self._settings["language_boost"] = service_lang + + # Validate language-model compatibility + # Filipino, Tamil, Persian only supported by speech-2.6-* models + if params.language in {Language.FA, Language.FIL, Language.TA}: + if not model.startswith("speech-2.6"): + logger.warning( + f"Language {params.language.value} ({service_lang}) is only supported by " + f"speech-2.6-hd and speech-2.6-turbo models. " + f"Current model '{model}' may not support this language. " + f"Consider using 'speech-2.6-turbo' or 'speech-2.6-hd'." + ) # Add optional emotion if provided if params.emotion: - # Validate emotion is in the supported list + # Validate emotion is in the supported list (updated per official docs) supported_emotions = [ "happy", "sad", @@ -176,15 +227,58 @@ class MiniMaxHttpTTSService(TTSService): "disgusted", "surprised", "neutral", + "fluent", ] if params.emotion in supported_emotions: self._settings["voice_setting"]["emotion"] = params.emotion else: - logger.warning(f"Unsupported emotion: {params.emotion}. Using default.") + logger.warning( + f"Unsupported emotion: {params.emotion}. Supported emotions: {supported_emotions}" + ) - # Add english_normalization if provided - if params.english_normalization is not None: - self._settings["english_normalization"] = params.english_normalization + # Add text_normalization if provided (corrected parameter name) + if params.text_normalization is not None: + self._settings["voice_setting"]["text_normalization"] = params.text_normalization + + # Add latex_read if provided + if params.latex_read is not None: + self._settings["voice_setting"]["latex_read"] = params.latex_read + + # Add force_cbr if provided (for MP3 format only) + if params.force_cbr is not None: + self._settings["audio_setting"]["force_cbr"] = params.force_cbr + + # Add subtitle settings if provided + if params.subtitle_enable is not None: + # Note: subtitle_enable only works in non-streaming mode (stream=false) + # Current implementation uses streaming mode (stream=true) + if params.subtitle_enable: + logger.warning( + "subtitle_enable is set to True, but this service uses streaming mode. " + "Subtitle generation (subtitle_enable) only works in non-streaming mode. " + "Subtitles may not be generated. " + "For subtitle support, consider implementing a non-streaming TTS service." + ) + + self._settings["subtitle_enable"] = params.subtitle_enable + + # Add subtitle_type only when subtitle_enable is True + if params.subtitle_enable and params.subtitle_type: + # Validate subtitle_type + if params.subtitle_type not in ["word", "sentence"]: + logger.warning( + f"Invalid subtitle_type: {params.subtitle_type}. " + f"Must be 'word' or 'sentence'. Using default 'sentence'." + ) + self._settings["subtitle_type"] = "sentence" + else: + self._settings["subtitle_type"] = params.subtitle_type + elif not params.subtitle_enable and params.subtitle_type != "sentence": + # Warn if subtitle_type is set but subtitle_enable is False + logger.debug( + f"subtitle_type='{params.subtitle_type}' will be ignored because " + f"subtitle_enable is False." + ) def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -231,7 +325,7 @@ class MiniMaxHttpTTSService(TTSService): """ await super().start(frame) self._settings["audio_setting"]["sample_rate"] = self.sample_rate - logger.debug(f"MiniMax TTS initialized with sample rate: {self.sample_rate}") + logger.debug(f"MiniMax TTS initialized with sample_rate={self.sample_rate}") @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: @@ -245,6 +339,9 @@ class MiniMaxHttpTTSService(TTSService): """ logger.debug(f"{self}: Generating TTS [{text}]") + # Reset trace_id for new request + self._current_trace_id = None + headers = { "accept": "application/json, text/plain, */*", "Content-Type": "application/json", @@ -262,9 +359,46 @@ class MiniMaxHttpTTSService(TTSService): async with self._session.post( self._base_url, headers=headers, json=payload ) as response: + # Extract trace_id from response header (available in all responses) + trace_id = response.headers.get("Trace-Id") or response.headers.get("trace-id") or response.headers.get("X-Trace-Id") or "unknown" + self._current_trace_id = trace_id + + # Log trace_id for all requests + logger.info(f"MiniMax TTS request trace_id={trace_id}, status={response.status}") + if response.status != 200: - error_message = f"MiniMax TTS error: HTTP {response.status}" - logger.error(error_message) + # Try to read error response body + try: + error_body = await response.text() + error_data = json.loads(error_body) + + # Extract MiniMax error details from body + base_resp = error_data.get("base_resp", {}) + status_code = base_resp.get("status_code", response.status) + status_msg = base_resp.get("status_msg", "Unknown error") + + error_message = ( + f"MiniMax TTS error: HTTP {response.status}, " + f"status_code={status_code}, status_msg={status_msg}, " + f"trace_id={trace_id}" + ) + logger.error( + error_message, + extra={ + "trace_id": trace_id, + "http_status": response.status, + "status_code": status_code, + "text_length": len(text), + }, + ) + except Exception as parse_error: + # If parsing fails, use basic error message + error_message = f"MiniMax TTS error: HTTP {response.status}, trace_id={trace_id}" + logger.error( + error_message, + extra={"http_status": response.status, "trace_id": trace_id, "parse_error": str(parse_error)}, + ) + yield ErrorFrame(error=error_message) return @@ -272,15 +406,49 @@ class MiniMaxHttpTTSService(TTSService): yield TTSStartedFrame() # Process the streaming response + logger.debug(f"Starting to read streaming response, status={response.status}, trace_id={trace_id}") buffer = bytearray() CHUNK_SIZE = self.chunk_size + chunk_count = 0 async for chunk in response.content.iter_chunked(CHUNK_SIZE): + chunk_count += 1 + logger.debug(f"Received chunk #{chunk_count}, size={len(chunk)} bytes") if not chunk: continue buffer.extend(chunk) + + # Log raw buffer content for debugging + if chunk_count == 1: + logger.debug(f"Raw buffer content: {buffer[:200]}") # First 200 bytes + + # Check if first chunk is a direct JSON error (not streaming format) + if not buffer.startswith(b"data:"): + try: + error_data = json.loads(buffer.decode("utf-8")) + base_resp = error_data.get("base_resp", {}) + status_code = base_resp.get("status_code", 0) + + if status_code != 0: + # This is a non-streaming error response + # Use trace_id from header (already extracted above) + status_msg = base_resp.get("status_msg", "Unknown error") + + error_message = ( + f"MiniMax TTS API error: status_code={status_code}, " + f"status_msg={status_msg}, trace_id={self._current_trace_id}" + ) + logger.error( + error_message, + extra={"trace_id": self._current_trace_id, "status_code": status_code}, + ) + yield ErrorFrame(error=error_message) + return + except (json.JSONDecodeError, UnicodeDecodeError): + # Not a valid JSON, continue with streaming processing + pass # Find complete data blocks while b"data:" in buffer: @@ -298,16 +466,56 @@ class MiniMaxHttpTTSService(TTSService): buffer = buffer[next_start:] try: - data = json.loads(data_block[5:].decode("utf-8")) - # Skip data blocks containing extra_info - if "extra_info" in data: - logger.debug("Received final chunk with extra info") - continue + data_str = data_block[5:].decode("utf-8") + logger.debug(f"Parsing data block: {data_str[:200]}...") # Log first 200 chars + data = json.loads(data_str) + # Check for business errors in base_resp + base_resp = data.get("base_resp", {}) + status_code = base_resp.get("status_code", 0) + + if status_code != 0: + # API returned business error + status_msg = base_resp.get("status_msg", "Unknown error") + error_message = ( + f"MiniMax TTS API error: status_code={status_code}, " + f"status_msg={status_msg}, trace_id={self._current_trace_id}" + ) + logger.error( + error_message, + extra={"trace_id": self._current_trace_id, "status_code": status_code}, + ) + yield ErrorFrame(error=error_message) + return + + # Handle final chunk with extra_info + if "extra_info" in data: + extra_info = data.get("extra_info", {}) + logger.info( + f"MiniMax TTS completed successfully, trace_id={self._current_trace_id}", + extra={ + "trace_id": self._current_trace_id, + "audio_length": extra_info.get("audio_length"), + "audio_size": extra_info.get("audio_size"), + "usage_characters": extra_info.get("usage_characters"), + "word_count": extra_info.get("word_count"), + }, + ) + continue # No audio data in this block + + # Extract audio data chunk_data = data.get("data", {}) if not chunk_data: continue + # Check for subtitle file (if subtitle generation is enabled) + subtitle_file = chunk_data.get("subtitle_file") + if subtitle_file: + logger.info( + f"Subtitle file available: {subtitle_file}", + extra={"trace_id": self._current_trace_id, "subtitle_url": subtitle_file}, + ) + audio_data = chunk_data.get("audio") if not audio_data: continue @@ -320,7 +528,7 @@ class MiniMaxHttpTTSService(TTSService): continue try: - # Convert this chunk of data + # Convert hex to binary audio_chunk = bytes.fromhex(hex_chunk) if audio_chunk: await self.stop_ttfb_metrics() @@ -330,16 +538,33 @@ class MiniMaxHttpTTSService(TTSService): num_channels=1, ) except ValueError as e: - logger.error(f"Error converting hex to binary: {e}") + logger.error( + f"Error converting hex to binary: {e}", + extra={"trace_id": self._current_trace_id}, + ) continue except json.JSONDecodeError as e: - logger.error(f"Error decoding JSON: {e}, data: {data_block[:100]}") + logger.error( + f"Error decoding JSON: {e}, data: {data_block[:100]}", + extra={"trace_id": self._current_trace_id or "unknown"}, + ) continue + except aiohttp.ClientError as e: + error_msg = f"MiniMax TTS network error: {str(e)}" + logger.exception(error_msg, extra={"trace_id": self._current_trace_id or "unknown"}) + yield ErrorFrame(error=error_msg) except Exception as e: logger.error(f"{self} exception: {e}") yield ErrorFrame(error=f"{self} error: {e}") finally: + if self._current_trace_id: + logger.debug( + f"MiniMax TTS request finished, trace_id={self._current_trace_id}, " + f"received {chunk_count if 'chunk_count' in locals() else 0} chunks" + ) + else: + logger.debug(f"MiniMax TTS request finished with no trace_id (no data received)") await self.stop_ttfb_metrics() yield TTSStoppedFrame() From 616e6ba351e015539aee8f597c8b51ea187a29fd Mon Sep 17 00:00:00 2001 From: minimax Date: Tue, 4 Nov 2025 10:52:13 +0800 Subject: [PATCH 080/110] docs(minimax): add API endpoint comment for west US region --- src/pipecat/services/minimax/tts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 8b7f71179..259144046 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -135,7 +135,9 @@ class MiniMaxHttpTTSService(TTSService): self, *, api_key: str, - base_url: str = "https://api.minimax.io/v1/t2a_v2", + base_url: str = "https://api.minimax.io/v1/t2a_v2", + # https://api-uw.minimax.io/v1/t2a_v2 + # support west of unite state group_id: str, model: str = "speech-02-turbo", voice_id: str = "Calm_Woman", From 17cf6c56cfbf5e36947708e2ff97f20e810f7729 Mon Sep 17 00:00:00 2001 From: vipyne Date: Tue, 4 Nov 2025 15:40:38 -0600 Subject: [PATCH 081/110] minimax updates some `debug`s -> `trace`s add western US base_url to docs ensure error_message is defined add deprecation warning for `english_normalization` param --- CHANGELOG.md | 6 ++ src/pipecat/__init__.py | 7 +- src/pipecat/services/minimax/tts.py | 118 ++++++++++------------------ 3 files changed, 48 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9887968dc..2cd406c0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -241,6 +241,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removed `needs_mcp_alternate_schema()` from `LLMService`. The mechanism that relied on it went away. +- In `MiniMaxHttpTTSService`: +-- Added support for speech-2.6-hd and speech-2.6-turbo models +-- Added languages: Afrikaans, Bulgarian, Catalan, Danish, Persian, Filipino, Hebrew, +Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and Tamil +-- Added new emotions: calm and fluent + ### Fixed - Restore backwards compatibility for vision/image features (broken in 0.0.92) diff --git a/src/pipecat/__init__.py b/src/pipecat/__init__.py index ed9892e9d..1975e1654 100644 --- a/src/pipecat/__init__.py +++ b/src/pipecat/__init__.py @@ -5,14 +5,11 @@ # import sys -from importlib.metadata import version, PackageNotFoundError +from importlib.metadata import version from loguru import logger -try: - __version__ = version("pipecat-ai") -except PackageNotFoundError: - __version__ = "0.0.0.dev" # Development version +__version__ = version("pipecat-ai") logger.info(f"ᓚᘏᗢ Pipecat {__version__} (Python {sys.version}) ᓚᘏᗢ") diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 259144046..22bb2f3c5 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -81,7 +81,7 @@ def language_to_minimax_language(language: Language) -> Optional[str]: Language.YUE: "Chinese,Yue", Language.ZH: "Chinese", } - + # Languages that require speech-2.6-* models V26_ONLY_LANGUAGES = {Language.FA, Language.FIL, Language.TA} @@ -110,6 +110,7 @@ class MiniMaxHttpTTSService(TTSService): pitch: Pitch adjustment (range: -12 to 12). emotion: Emotional tone (options: "happy", "sad", "angry", "fearful", "disgusted", "surprised", "calm", "fluent"). + english_normalization: Deprecated; use `text_normalization` instead text_normalization: Enable text normalization (Chinese/English). latex_read: Enable LaTeX formula reading. force_cbr: Enable Constant Bitrate (CBR) for audio encoding (MP3 only). @@ -124,6 +125,7 @@ class MiniMaxHttpTTSService(TTSService): volume: Optional[float] = 1.0 pitch: Optional[int] = 0 emotion: Optional[str] = None + english_normalization: Optional[bool] = None # Deprecated text_normalization: Optional[bool] = None latex_read: Optional[bool] = None force_cbr: Optional[bool] = None @@ -135,9 +137,7 @@ class MiniMaxHttpTTSService(TTSService): self, *, api_key: str, - base_url: str = "https://api.minimax.io/v1/t2a_v2", - # https://api-uw.minimax.io/v1/t2a_v2 - # support west of unite state + base_url: str = "https://api.minimax.io/v1/t2a_v2", group_id: str, model: str = "speech-02-turbo", voice_id: str = "Calm_Woman", @@ -153,6 +153,7 @@ class MiniMaxHttpTTSService(TTSService): base_url: API base URL, defaults to MiniMax's T2A endpoint. Global: https://api.minimax.io/v1/t2a_v2 Mainland China: https://api.minimaxi.chat/v1/t2a_v2 + Western United States: https://api-uw.minimax.io/v1/t2a_v2 group_id: MiniMax Group ID to identify project. model: TTS model name. Defaults to "speech-02-turbo". Options include: "speech-2.6-hd", "speech-2.6-turbo" (latest, supports Filipino/Tamil/Persian), @@ -174,7 +175,6 @@ class MiniMaxHttpTTSService(TTSService): self._session = aiohttp_session self._model_name = model self._voice_id = voice_id - self._current_trace_id: Optional[str] = None # Create voice settings self._settings = { @@ -206,7 +206,7 @@ class MiniMaxHttpTTSService(TTSService): service_lang = self.language_to_service_language(params.language) if service_lang: self._settings["language_boost"] = service_lang - + # Validate language-model compatibility # Filipino, Tamil, Persian only supported by speech-2.6-* models if params.language in {Language.FA, Language.FIL, Language.TA}: @@ -238,6 +238,13 @@ class MiniMaxHttpTTSService(TTSService): f"Unsupported emotion: {params.emotion}. Supported emotions: {supported_emotions}" ) + # If `english_normalization`, add `text_normalization` and print warning + if params.english_normalization is not None: + logger.warning( + "Parameter `english_normalization` is deprecated and will be removed in a future version. Use `text_normalization` instead." + ) + self._settings["voice_setting"]["text_normalization"] = params.english_normalization + # Add text_normalization if provided (corrected parameter name) if params.text_normalization is not None: self._settings["voice_setting"]["text_normalization"] = params.text_normalization @@ -261,9 +268,9 @@ class MiniMaxHttpTTSService(TTSService): "Subtitles may not be generated. " "For subtitle support, consider implementing a non-streaming TTS service." ) - + self._settings["subtitle_enable"] = params.subtitle_enable - + # Add subtitle_type only when subtitle_enable is True if params.subtitle_enable and params.subtitle_type: # Validate subtitle_type @@ -341,9 +348,6 @@ class MiniMaxHttpTTSService(TTSService): """ logger.debug(f"{self}: Generating TTS [{text}]") - # Reset trace_id for new request - self._current_trace_id = None - headers = { "accept": "application/json, text/plain, */*", "Content-Type": "application/json", @@ -361,14 +365,10 @@ class MiniMaxHttpTTSService(TTSService): async with self._session.post( self._base_url, headers=headers, json=payload ) as response: - # Extract trace_id from response header (available in all responses) - trace_id = response.headers.get("Trace-Id") or response.headers.get("trace-id") or response.headers.get("X-Trace-Id") or "unknown" - self._current_trace_id = trace_id - - # Log trace_id for all requests - logger.info(f"MiniMax TTS request trace_id={trace_id}, status={response.status}") - + logger.trace(f"MiniMax TTS request status={response.status}") + if response.status != 200: + error_message = "MiniMax TTS error" # Try to read error response body try: error_body = await response.text() @@ -381,25 +381,13 @@ class MiniMaxHttpTTSService(TTSService): error_message = ( f"MiniMax TTS error: HTTP {response.status}, " - f"status_code={status_code}, status_msg={status_msg}, " - f"trace_id={trace_id}" - ) - logger.error( - error_message, - extra={ - "trace_id": trace_id, - "http_status": response.status, - "status_code": status_code, - "text_length": len(text), - }, + f"status_code={status_code}, status_msg={status_msg}" ) + logger.error(error_message) except Exception as parse_error: # If parsing fails, use basic error message - error_message = f"MiniMax TTS error: HTTP {response.status}, trace_id={trace_id}" - logger.error( - error_message, - extra={"http_status": response.status, "trace_id": trace_id, "parse_error": str(parse_error)}, - ) + error_message = f"MiniMax TTS error: HTTP {response.status}" + logger.error(error_message) yield ErrorFrame(error=error_message) return @@ -408,7 +396,7 @@ class MiniMaxHttpTTSService(TTSService): yield TTSStartedFrame() # Process the streaming response - logger.debug(f"Starting to read streaming response, status={response.status}, trace_id={trace_id}") + logger.trace(f"Starting to read streaming response, status={response.status}") buffer = bytearray() CHUNK_SIZE = self.chunk_size @@ -416,36 +404,32 @@ class MiniMaxHttpTTSService(TTSService): async for chunk in response.content.iter_chunked(CHUNK_SIZE): chunk_count += 1 - logger.debug(f"Received chunk #{chunk_count}, size={len(chunk)} bytes") + logger.trace(f"Received chunk #{chunk_count}, size={len(chunk)} bytes") if not chunk: continue buffer.extend(chunk) - + # Log raw buffer content for debugging if chunk_count == 1: - logger.debug(f"Raw buffer content: {buffer[:200]}") # First 200 bytes - + logger.trace(f"Raw buffer content: {buffer[:200]}") # First 200 bytes + # Check if first chunk is a direct JSON error (not streaming format) if not buffer.startswith(b"data:"): try: error_data = json.loads(buffer.decode("utf-8")) base_resp = error_data.get("base_resp", {}) status_code = base_resp.get("status_code", 0) - + if status_code != 0: # This is a non-streaming error response - # Use trace_id from header (already extracted above) status_msg = base_resp.get("status_msg", "Unknown error") - + error_message = ( - f"MiniMax TTS API error: status_code={status_code}, " - f"status_msg={status_msg}, trace_id={self._current_trace_id}" - ) - logger.error( - error_message, - extra={"trace_id": self._current_trace_id, "status_code": status_code}, + f"MiniMax TTS API error: status_code={status_code}" + f"status_msg={status_msg}" ) + logger.error(error_message) yield ErrorFrame(error=error_message) return except (json.JSONDecodeError, UnicodeDecodeError): @@ -469,7 +453,9 @@ class MiniMaxHttpTTSService(TTSService): try: data_str = data_block[5:].decode("utf-8") - logger.debug(f"Parsing data block: {data_str[:200]}...") # Log first 200 chars + logger.trace( + f"Parsing data block: {data_str[:200]}..." + ) # Log first 200 chars data = json.loads(data_str) # Check for business errors in base_resp @@ -481,28 +467,16 @@ class MiniMaxHttpTTSService(TTSService): status_msg = base_resp.get("status_msg", "Unknown error") error_message = ( f"MiniMax TTS API error: status_code={status_code}, " - f"status_msg={status_msg}, trace_id={self._current_trace_id}" - ) - logger.error( - error_message, - extra={"trace_id": self._current_trace_id, "status_code": status_code}, + f"status_msg={status_msg}" ) + logger.error(error_message) yield ErrorFrame(error=error_message) return # Handle final chunk with extra_info if "extra_info" in data: extra_info = data.get("extra_info", {}) - logger.info( - f"MiniMax TTS completed successfully, trace_id={self._current_trace_id}", - extra={ - "trace_id": self._current_trace_id, - "audio_length": extra_info.get("audio_length"), - "audio_size": extra_info.get("audio_size"), - "usage_characters": extra_info.get("usage_characters"), - "word_count": extra_info.get("word_count"), - }, - ) + logger.debug(f"Received final chunk with extra info: {extra_info}") continue # No audio data in this block # Extract audio data @@ -513,10 +487,7 @@ class MiniMaxHttpTTSService(TTSService): # Check for subtitle file (if subtitle generation is enabled) subtitle_file = chunk_data.get("subtitle_file") if subtitle_file: - logger.info( - f"Subtitle file available: {subtitle_file}", - extra={"trace_id": self._current_trace_id, "subtitle_url": subtitle_file}, - ) + logger.info(f"Subtitle file available: {subtitle_file}") audio_data = chunk_data.get("audio") if not audio_data: @@ -542,31 +513,22 @@ class MiniMaxHttpTTSService(TTSService): except ValueError as e: logger.error( f"Error converting hex to binary: {e}", - extra={"trace_id": self._current_trace_id}, ) continue except json.JSONDecodeError as e: logger.error( f"Error decoding JSON: {e}, data: {data_block[:100]}", - extra={"trace_id": self._current_trace_id or "unknown"}, ) continue except aiohttp.ClientError as e: error_msg = f"MiniMax TTS network error: {str(e)}" - logger.exception(error_msg, extra={"trace_id": self._current_trace_id or "unknown"}) + logger.exception(error_msg) yield ErrorFrame(error=error_msg) except Exception as e: logger.error(f"{self} exception: {e}") yield ErrorFrame(error=f"{self} error: {e}") finally: - if self._current_trace_id: - logger.debug( - f"MiniMax TTS request finished, trace_id={self._current_trace_id}, " - f"received {chunk_count if 'chunk_count' in locals() else 0} chunks" - ) - else: - logger.debug(f"MiniMax TTS request finished with no trace_id (no data received)") await self.stop_ttfb_metrics() yield TTSStoppedFrame() From 59d40eac4516006ac56d098bc10d53ddb4986904 Mon Sep 17 00:00:00 2001 From: Vanessa Pyne Date: Wed, 5 Nov 2025 09:11:55 -0600 Subject: [PATCH 082/110] Update src/pipecat/services/minimax/tts.py Co-authored-by: Mark Backman add warning --- src/pipecat/services/minimax/tts.py | 59 ++++++++--------------------- 1 file changed, 15 insertions(+), 44 deletions(-) diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 22bb2f3c5..42e70cd50 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -111,9 +111,13 @@ class MiniMaxHttpTTSService(TTSService): emotion: Emotional tone (options: "happy", "sad", "angry", "fearful", "disgusted", "surprised", "calm", "fluent"). english_normalization: Deprecated; use `text_normalization` instead + + .. deprecated:: 0.0.93 + The `english_normalization` parameter is deprecated and will be removed in a future version. + Use the `text_normalization` parameter instead. + text_normalization: Enable text normalization (Chinese/English). latex_read: Enable LaTeX formula reading. - force_cbr: Enable Constant Bitrate (CBR) for audio encoding (MP3 only). exclude_aggregated_audio: Whether to exclude aggregated audio in final chunk. subtitle_enable: Enable subtitle generation (non-streaming only). subtitle_type: Subtitle timestamp granularity (options: "word", "sentence"). @@ -128,7 +132,6 @@ class MiniMaxHttpTTSService(TTSService): english_normalization: Optional[bool] = None # Deprecated text_normalization: Optional[bool] = None latex_read: Optional[bool] = None - force_cbr: Optional[bool] = None exclude_aggregated_audio: Optional[bool] = None subtitle_enable: Optional[bool] = None subtitle_type: Optional[str] = "sentence" @@ -207,20 +210,9 @@ class MiniMaxHttpTTSService(TTSService): if service_lang: self._settings["language_boost"] = service_lang - # Validate language-model compatibility - # Filipino, Tamil, Persian only supported by speech-2.6-* models - if params.language in {Language.FA, Language.FIL, Language.TA}: - if not model.startswith("speech-2.6"): - logger.warning( - f"Language {params.language.value} ({service_lang}) is only supported by " - f"speech-2.6-hd and speech-2.6-turbo models. " - f"Current model '{model}' may not support this language. " - f"Consider using 'speech-2.6-turbo' or 'speech-2.6-hd'." - ) - # Add optional emotion if provided if params.emotion: - # Validate emotion is in the supported list (updated per official docs) + # Validate emotion is in the supported list supported_emotions = [ "happy", "sad", @@ -240,9 +232,14 @@ class MiniMaxHttpTTSService(TTSService): # If `english_normalization`, add `text_normalization` and print warning if params.english_normalization is not None: - logger.warning( - "Parameter `english_normalization` is deprecated and will be removed in a future version. Use `text_normalization` instead." - ) + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Parameter `english_normalization` is deprecated and will be removed in a future version. Use `text_normalization` instead.", + DeprecationWarning, + ) self._settings["voice_setting"]["text_normalization"] = params.english_normalization # Add text_normalization if provided (corrected parameter name) @@ -253,10 +250,6 @@ class MiniMaxHttpTTSService(TTSService): if params.latex_read is not None: self._settings["voice_setting"]["latex_read"] = params.latex_read - # Add force_cbr if provided (for MP3 format only) - if params.force_cbr is not None: - self._settings["audio_setting"]["force_cbr"] = params.force_cbr - # Add subtitle settings if provided if params.subtitle_enable is not None: # Note: subtitle_enable only works in non-streaming mode (stream=false) @@ -334,7 +327,7 @@ class MiniMaxHttpTTSService(TTSService): """ await super().start(frame) self._settings["audio_setting"]["sample_rate"] = self.sample_rate - logger.debug(f"MiniMax TTS initialized with sample_rate={self.sample_rate}") + logger.debug(f"MiniMax TTS initialized with sample_rate: {self.sample_rate}") @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: @@ -414,28 +407,6 @@ class MiniMaxHttpTTSService(TTSService): if chunk_count == 1: logger.trace(f"Raw buffer content: {buffer[:200]}") # First 200 bytes - # Check if first chunk is a direct JSON error (not streaming format) - if not buffer.startswith(b"data:"): - try: - error_data = json.loads(buffer.decode("utf-8")) - base_resp = error_data.get("base_resp", {}) - status_code = base_resp.get("status_code", 0) - - if status_code != 0: - # This is a non-streaming error response - status_msg = base_resp.get("status_msg", "Unknown error") - - error_message = ( - f"MiniMax TTS API error: status_code={status_code}" - f"status_msg={status_msg}" - ) - logger.error(error_message) - yield ErrorFrame(error=error_message) - return - except (json.JSONDecodeError, UnicodeDecodeError): - # Not a valid JSON, continue with streaming processing - pass - # Find complete data blocks while b"data:" in buffer: start = buffer.find(b"data:") From 06542a2dbc3b035cc1c44491547b25a3a53fd194 Mon Sep 17 00:00:00 2001 From: vipyne Date: Wed, 19 Nov 2025 11:02:13 -0600 Subject: [PATCH 083/110] Update CHANGELOG --- CHANGELOG.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cd406c0c..843fd98c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,11 +78,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated language mappings for the Google and Gemini TTS services to match official documentation. +- In `MiniMaxHttpTTSService`: +-- Added support for speech-2.6-hd and speech-2.6-turbo models +-- Added languages: Afrikaans, Bulgarian, Catalan, Danish, Persian, Filipino, Hebrew, +Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and Tamil +-- Added new emotions: calm and fluent + ### Deprecated - The `api_key` parameter in `GeminiTTSService` is deprecated. Use `credentials` or `credentials_path` instead for Google Cloud authentication. +- `english_normalization` input parameter for `MiniMaxHttpTTSService` is deprecated, +use `test_normalization` instead. + ### Fixed - Fixed a `SimliVideoService` connection issue. @@ -241,12 +250,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removed `needs_mcp_alternate_schema()` from `LLMService`. The mechanism that relied on it went away. -- In `MiniMaxHttpTTSService`: --- Added support for speech-2.6-hd and speech-2.6-turbo models --- Added languages: Afrikaans, Bulgarian, Catalan, Danish, Persian, Filipino, Hebrew, -Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and Tamil --- Added new emotions: calm and fluent - ### Fixed - Restore backwards compatibility for vision/image features (broken in 0.0.92) From 954849379b62fea25c70a1361af33cd220f40bf0 Mon Sep 17 00:00:00 2001 From: vipyne Date: Wed, 19 Nov 2025 11:10:27 -0600 Subject: [PATCH 084/110] cleanup --- src/pipecat/services/minimax/tts.py | 122 ++-------------------------- 1 file changed, 8 insertions(+), 114 deletions(-) diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 42e70cd50..4cd86fac3 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -82,9 +82,6 @@ def language_to_minimax_language(language: Language) -> Optional[str]: Language.ZH: "Chinese", } - # Languages that require speech-2.6-* models - V26_ONLY_LANGUAGES = {Language.FA, Language.FIL, Language.TA} - return resolve_language(language, LANGUAGE_MAP, use_base_code=False) @@ -112,16 +109,13 @@ class MiniMaxHttpTTSService(TTSService): "disgusted", "surprised", "calm", "fluent"). english_normalization: Deprecated; use `text_normalization` instead - .. deprecated:: 0.0.93 + .. deprecated:: 0.0.96 The `english_normalization` parameter is deprecated and will be removed in a future version. Use the `text_normalization` parameter instead. text_normalization: Enable text normalization (Chinese/English). latex_read: Enable LaTeX formula reading. exclude_aggregated_audio: Whether to exclude aggregated audio in final chunk. - subtitle_enable: Enable subtitle generation (non-streaming only). - subtitle_type: Subtitle timestamp granularity (options: "word", "sentence"). - Only effective when subtitle_enable is True. Defaults to "sentence". """ language: Optional[Language] = Language.EN @@ -133,8 +127,6 @@ class MiniMaxHttpTTSService(TTSService): text_normalization: Optional[bool] = None latex_read: Optional[bool] = None exclude_aggregated_audio: Optional[bool] = None - subtitle_enable: Optional[bool] = None - subtitle_type: Optional[str] = "sentence" def __init__( self, @@ -194,12 +186,6 @@ class MiniMaxHttpTTSService(TTSService): }, } - # Add stream_options if exclude_aggregated_audio is set - if params.exclude_aggregated_audio is not None: - self._settings["stream_options"] = { - "exclude_aggregated_audio": params.exclude_aggregated_audio - } - # Set voice and model self.set_voice(voice_id) self.set_model_name(model) @@ -250,38 +236,6 @@ class MiniMaxHttpTTSService(TTSService): if params.latex_read is not None: self._settings["voice_setting"]["latex_read"] = params.latex_read - # Add subtitle settings if provided - if params.subtitle_enable is not None: - # Note: subtitle_enable only works in non-streaming mode (stream=false) - # Current implementation uses streaming mode (stream=true) - if params.subtitle_enable: - logger.warning( - "subtitle_enable is set to True, but this service uses streaming mode. " - "Subtitle generation (subtitle_enable) only works in non-streaming mode. " - "Subtitles may not be generated. " - "For subtitle support, consider implementing a non-streaming TTS service." - ) - - self._settings["subtitle_enable"] = params.subtitle_enable - - # Add subtitle_type only when subtitle_enable is True - if params.subtitle_enable and params.subtitle_type: - # Validate subtitle_type - if params.subtitle_type not in ["word", "sentence"]: - logger.warning( - f"Invalid subtitle_type: {params.subtitle_type}. " - f"Must be 'word' or 'sentence'. Using default 'sentence'." - ) - self._settings["subtitle_type"] = "sentence" - else: - self._settings["subtitle_type"] = params.subtitle_type - elif not params.subtitle_enable and params.subtitle_type != "sentence": - # Warn if subtitle_type is set but subtitle_enable is False - logger.debug( - f"subtitle_type='{params.subtitle_type}' will be ignored because " - f"subtitle_enable is False." - ) - def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -358,30 +312,9 @@ class MiniMaxHttpTTSService(TTSService): async with self._session.post( self._base_url, headers=headers, json=payload ) as response: - logger.trace(f"MiniMax TTS request status={response.status}") - if response.status != 200: - error_message = "MiniMax TTS error" - # Try to read error response body - try: - error_body = await response.text() - error_data = json.loads(error_body) - - # Extract MiniMax error details from body - base_resp = error_data.get("base_resp", {}) - status_code = base_resp.get("status_code", response.status) - status_msg = base_resp.get("status_msg", "Unknown error") - - error_message = ( - f"MiniMax TTS error: HTTP {response.status}, " - f"status_code={status_code}, status_msg={status_msg}" - ) - logger.error(error_message) - except Exception as parse_error: - # If parsing fails, use basic error message - error_message = f"MiniMax TTS error: HTTP {response.status}" - logger.error(error_message) - + error_message = f"MiniMax TTS error: HTTP {response.status}" + logger.error(error_message) yield ErrorFrame(error=error_message) return @@ -389,24 +322,16 @@ class MiniMaxHttpTTSService(TTSService): yield TTSStartedFrame() # Process the streaming response - logger.trace(f"Starting to read streaming response, status={response.status}") buffer = bytearray() CHUNK_SIZE = self.chunk_size - chunk_count = 0 async for chunk in response.content.iter_chunked(CHUNK_SIZE): - chunk_count += 1 - logger.trace(f"Received chunk #{chunk_count}, size={len(chunk)} bytes") if not chunk: continue buffer.extend(chunk) - # Log raw buffer content for debugging - if chunk_count == 1: - logger.trace(f"Raw buffer content: {buffer[:200]}") # First 200 bytes - # Find complete data blocks while b"data:" in buffer: start = buffer.find(b"data:") @@ -423,43 +348,16 @@ class MiniMaxHttpTTSService(TTSService): buffer = buffer[next_start:] try: - data_str = data_block[5:].decode("utf-8") - logger.trace( - f"Parsing data block: {data_str[:200]}..." - ) # Log first 200 chars - data = json.loads(data_str) - - # Check for business errors in base_resp - base_resp = data.get("base_resp", {}) - status_code = base_resp.get("status_code", 0) - - if status_code != 0: - # API returned business error - status_msg = base_resp.get("status_msg", "Unknown error") - error_message = ( - f"MiniMax TTS API error: status_code={status_code}, " - f"status_msg={status_msg}" - ) - logger.error(error_message) - yield ErrorFrame(error=error_message) - return - - # Handle final chunk with extra_info + data = json.loads(data_block[5:].decode("utf-8")) + # Skip data blocks containing extra_info if "extra_info" in data: - extra_info = data.get("extra_info", {}) - logger.debug(f"Received final chunk with extra info: {extra_info}") - continue # No audio data in this block + logger.debug("Received final chunk with extra info") + continue - # Extract audio data chunk_data = data.get("data", {}) if not chunk_data: continue - # Check for subtitle file (if subtitle generation is enabled) - subtitle_file = chunk_data.get("subtitle_file") - if subtitle_file: - logger.info(f"Subtitle file available: {subtitle_file}") - audio_data = chunk_data.get("audio") if not audio_data: continue @@ -472,7 +370,7 @@ class MiniMaxHttpTTSService(TTSService): continue try: - # Convert hex to binary + # Convert this chunk of data audio_chunk = bytes.fromhex(hex_chunk) if audio_chunk: await self.stop_ttfb_metrics() @@ -493,10 +391,6 @@ class MiniMaxHttpTTSService(TTSService): ) continue - except aiohttp.ClientError as e: - error_msg = f"MiniMax TTS network error: {str(e)}" - logger.exception(error_msg) - yield ErrorFrame(error=error_msg) except Exception as e: logger.error(f"{self} exception: {e}") yield ErrorFrame(error=f"{self} error: {e}") From ec8964425a39166b3372fca206fdfff84b4031f3 Mon Sep 17 00:00:00 2001 From: fbarril Date: Fri, 21 Nov 2025 00:27:57 +0000 Subject: [PATCH 085/110] add livekit helper --- pyproject.toml | 1 + src/pipecat/transports/livekit/utils.py | 96 +++++++++++++++++++++++++ uv.lock | 6 +- 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 src/pipecat/transports/livekit/utils.py diff --git a/pyproject.toml b/pyproject.toml index 73da9083a..149bf18a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,7 @@ webrtc = [ "aiortc>=1.13.0,<2", "opencv-python>=4.11.0.86,<5" ] websocket = [ "pipecat-ai[websockets-base]", "fastapi>=0.115.6,<0.122.0" ] websockets-base = [ "websockets>=13.1,<16.0" ] whisper = [ "faster-whisper~=1.1.1" ] +auth = [ "pyjwt>=2.10.1" ] [dependency-groups] dev = [ diff --git a/src/pipecat/transports/livekit/utils.py b/src/pipecat/transports/livekit/utils.py new file mode 100644 index 000000000..741b820b2 --- /dev/null +++ b/src/pipecat/transports/livekit/utils.py @@ -0,0 +1,96 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""LiveKit REST Helpers. + +Methods that wrap the LiveKit API for room management. +""" + +import aiohttp + + +class LiveKitRESTHelper: + """Helper class for interacting with LiveKit's REST API. + + Provides methods for managing LiveKit rooms. + """ + + def __init__( + self, + *, + api_key: str, + api_secret: str, + api_url: str = "https://your-livekit-host.com", + aiohttp_session: aiohttp.ClientSession, + ): + """Initialize the LiveKit REST helper. + + Args: + api_key: Your LiveKit API key. + api_secret: Your LiveKit API secret. + api_url: LiveKit server URL (e.g. "https://your-livekit-host.com"). + aiohttp_session: Async HTTP session for making requests. + """ + self.api_key = api_key + self.api_secret = api_secret + self.api_url = api_url.rstrip("/") + self.aiohttp_session = aiohttp_session + + def _create_access_token(self, room_create: bool = True) -> str: + """Create a signed access token for LiveKit API authentication. + + Args: + room_create: Whether to grant roomCreate permission. + + Returns: + Signed JWT access token. + """ + import time + + import jwt + + claims = { + "iss": self.api_key, + "sub": self.api_key, + "nbf": int(time.time()), + "exp": int(time.time()) + 60, # Token valid for 60 seconds + "video": { + "roomCreate": room_create, + }, + } + + return jwt.encode(claims, self.api_secret, algorithm="HS256") + + async def delete_room_by_name(self, room_name: str) -> bool: + """Delete a LiveKit room by name. + + This will forcibly disconnect all participants currently in the room. + + Args: + room_name: Name of the room to delete. + + Returns: + True if deletion was successful. + + Raises: + Exception: If deletion fails. + """ + token = self._create_access_token(room_create=True) + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + + async with self.aiohttp_session.post( + f"{self.api_url}/twirp/livekit.RoomService/DeleteRoom", + headers=headers, + json={"room": room_name}, + ) as r: + if r.status != 200: + text = await r.text() + raise Exception(f"Failed to delete room [{room_name}] (status: {r.status}): {text}") + + return True diff --git a/uv.lock b/uv.lock index 662150b5b..8f7585828 100644 --- a/uv.lock +++ b/uv.lock @@ -4442,6 +4442,9 @@ assemblyai = [ asyncai = [ { name = "websockets" }, ] +auth = [ + { name = "pyjwt" }, +] aws = [ { name = "aioboto3" }, { name = "websockets" }, @@ -4721,6 +4724,7 @@ requires-dist = [ { name = "pyaudio", marker = "extra == 'local'", specifier = "~=0.2.14" }, { name = "pydantic", specifier = ">=2.10.6,<3" }, { name = "pygobject", marker = "extra == 'gstreamer'", specifier = "~=3.50.0" }, + { name = "pyjwt", marker = "extra == 'auth'", specifier = ">=2.10.1" }, { name = "pyloudnorm", specifier = "~=0.1.1" }, { name = "python-dotenv", marker = "extra == 'runner'", specifier = ">=1.0.0,<2.0.0" }, { name = "pyvips", extras = ["binary"], marker = "extra == 'moondream'", specifier = "~=3.0.0" }, @@ -4745,7 +4749,7 @@ requires-dist = [ { name = "wait-for2", marker = "python_full_version < '3.12'", specifier = ">=0.4.1" }, { name = "websockets", marker = "extra == 'websockets-base'", specifier = ">=13.1,<16.0" }, ] -provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "cerebras", "deepseek", "daily", "deepgram", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "krisp", "koala", "langchain", "livekit", "lmnt", "local", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "nim", "neuphonic", "noisereduce", "openai", "openpipe", "openrouter", "perplexity", "playht", "qwen", "rime", "riva", "runner", "sambanova", "sarvam", "sentry", "local-smart-turn", "local-smart-turn-v3", "remote-smart-turn", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] +provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "cerebras", "deepseek", "daily", "deepgram", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "krisp", "koala", "langchain", "livekit", "lmnt", "local", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "nim", "neuphonic", "noisereduce", "openai", "openpipe", "openrouter", "perplexity", "playht", "qwen", "rime", "riva", "runner", "sambanova", "sarvam", "sentry", "local-smart-turn", "local-smart-turn-v3", "remote-smart-turn", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper", "auth"] [package.metadata.requires-dev] dev = [ From dcc20f86e184696bb90823be5277d6f2c4c6f736 Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Fri, 14 Nov 2025 18:11:55 -0500 Subject: [PATCH 086/110] Updated the BaseTextAggregator to categorize aggregations Modified the BaseTextAggregator type so that when text gets aggregated, metadata can be associated with it. Currently, that just means a `type`, so that the aggregation can be classified or described. Changes made to support this: - **IMPORTANT**: Aggregators are now expected to strip leading/trailing white space characters before returning their aggregation from `aggregation()` or `.text`. This way all aggregators have a consistent contract allowing downstream use to know how to stitch aggregations back together - Introduced a new `Aggregation` dataclass to represent both the aggregated `text` and a string identifying the `type` of aggregation (ex. "sentence", "word", "my custom aggregation") - **BREAKING**: `BaseTextAggregator.text` now returns an `Aggregation` (instead of `str`). To update: `aggregated_text = myAggregator.text` -> `aggregated_text = myAggregator.text.text` - **BREAKING**: `BaseTextAggregator.aggregate()` now returns `Optional[Aggregation]` (instead of `Optional[str]`). To update: ``` aggregation = myAggregator.aggregate(text) if (aggregation): print(f"successfully aggregated text: {aggregation.text}") // instead of {aggregation} ``` - `SimpleTextAggregator`, `SkipTagsAggregator`, `PatternPairAggregator` updated to produce/consume `Aggregation` objects. - All uses of the above Aggregators have been updated accordingly. --- CHANGELOG.md | 24 ++++++++ src/pipecat/extensions/ivr/ivr_navigator.py | 2 +- src/pipecat/services/tts_service.py | 29 ++++----- src/pipecat/tests/utils.py | 12 +++- .../utils/text/base_text_aggregator.py | 60 ++++++++++++++++--- .../utils/text/pattern_pair_aggregator.py | 15 ++--- .../utils/text/simple_text_aggregator.py | 12 ++-- .../utils/text/skip_tags_aggregator.py | 15 ++--- tests/test_pattern_pair_aggregator.py | 30 ++++++---- tests/test_simple_text_aggregator.py | 18 ++++-- tests/test_skip_tags_aggregator.py | 24 ++++---- 11 files changed, 166 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 843fd98c4..497e97f2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and Tamil -- Added new emotions: calm and fluent +- `BaseTextAggregator` changes: + Modified the BaseTextAggregator type so that when text gets aggregated, metadata can + be associated with it. Currently, that just means a `type`, so that the aggregation + can be classified or described. Changes made to support this: + - **IMPORTANT**: Aggregators are now expected to strip leading/trailing white space + characters before returning their aggregation from `aggregation()` or `.text`. This + way all aggregators have a consistent contract allowing downstream use to know how + to stitch aggregations back together. + - Introduced a new `Aggregation` dataclass to represent both the aggregated `text` and + a string identifying the `type` of aggregation (ex. "sentence", "word", "my custom + aggregation") + - **BREAKING**: `BaseTextAggregator.text` now returns an `Aggregation` (instead of `str`). + To update: `aggregated_text = myAggregator.text` -> `aggregated_text = myAggregator.text.text` + - **BREAKING**: `BaseTextAggregator.aggregate()` now returns `Optional[Aggregation]` + (instead of `Optional[str]`). To update: + ``` + aggregation = myAggregator.aggregate(text) + if (aggregation): + print(f"successfully aggregated text: {aggregation.text}") // instead of {aggregation} + ``` + - `SimpleTextAggregator`, `SkipTagsAggregator`, `PatternPairAggregator` updated to + produce/consume `Aggregation` objects. + - All uses of the above Aggregators have been updated accordingly. + ### Deprecated - The `api_key` parameter in `GeminiTTSService` is deprecated. Use diff --git a/src/pipecat/extensions/ivr/ivr_navigator.py b/src/pipecat/extensions/ivr/ivr_navigator.py index 05748d94f..9c59772ad 100644 --- a/src/pipecat/extensions/ivr/ivr_navigator.py +++ b/src/pipecat/extensions/ivr/ivr_navigator.py @@ -148,7 +148,7 @@ class IVRProcessor(FrameProcessor): result = await self._aggregator.aggregate(frame.text) if result: # Push aggregated text that doesn't contain XML patterns - await self.push_frame(LLMTextFrame(result), direction) + await self.push_frame(LLMTextFrame(result.text), direction) else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index f0d602a40..33cf7d103 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -142,7 +142,6 @@ class TTSService(AIService): self._voice_id: str = "" self._settings: Dict[str, Any] = {} self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() - self._aggregated_text_includes_inter_frame_spaces: bool = False self._text_filters: Sequence[BaseTextFilter] = text_filters or [] self._transport_destination: Optional[str] = transport_destination self._tracing_enabled: bool = False @@ -352,17 +351,14 @@ class TTSService(AIService): # pause to avoid audio overlapping. await self._maybe_pause_frame_processing() - sentence = self._text_aggregator.text - includes_inter_frame_spaces = self._aggregated_text_includes_inter_frame_spaces + pending_aggregation = self._text_aggregator.text # Reset aggregator state await self._text_aggregator.reset() self._processing_text = False - self._aggregated_text_includes_inter_frame_spaces = False - await self._push_tts_frames( - sentence, includes_inter_frame_spaces=includes_inter_frame_spaces - ) + if pending_aggregation.text: + await self._push_tts_frames(pending_aggregation.text) if isinstance(frame, LLMFullResponseEndFrame): if self._push_text_frames: await self.push_frame(frame, direction) @@ -372,7 +368,7 @@ class TTSService(AIService): # Store if we were processing text or not so we can set it back. processing_text = self._processing_text # Assumption: text in TTSSpeakFrame does not include inter-frame spaces - await self._push_tts_frames(frame.text, includes_inter_frame_spaces=False) + await self._push_tts_frames(frame.text) # We pause processing incoming frames because we are sending data to # the TTS. We pause to avoid audio overlapping. await self._maybe_pause_frame_processing() @@ -462,21 +458,20 @@ class TTSService(AIService): async def _process_text_frame(self, frame: TextFrame): text: Optional[str] = None + includes_inter_frame_spaces: bool = False if not self._aggregate_sentences: text = frame.text + includes_inter_frame_spaces = frame.includes_inter_frame_spaces else: - text = await self._text_aggregator.aggregate(frame.text) - # Assumption: whether inter-frame spaces are included shouldn't - # change during aggregation, so we can just use the latest frame's - # value - self._aggregated_text_includes_inter_frame_spaces = frame.includes_inter_frame_spaces + aggregation = await self._text_aggregator.aggregate(frame.text) + text = aggregation.text if text: - await self._push_tts_frames( - text, includes_inter_frame_spaces=frame.includes_inter_frame_spaces - ) + await self._push_tts_frames(text, includes_inter_frame_spaces) - async def _push_tts_frames(self, text: str, includes_inter_frame_spaces: bool): + async def _push_tts_frames( + self, text: str, includes_inter_frame_spaces: Optional[bool] = False + ): # Remove leading newlines only text = text.lstrip("\n") diff --git a/src/pipecat/tests/utils.py b/src/pipecat/tests/utils.py index 6ccce4b31..94b8cb1a4 100644 --- a/src/pipecat/tests/utils.py +++ b/src/pipecat/tests/utils.py @@ -203,8 +203,16 @@ async def run_test( if not isinstance(frame, EndFrame) or not send_end_frame: received_down_frames.append(frame) - print("received DOWN frames =", received_down_frames) - print("expected DOWN frames =", expected_down_frames) + down_frames_printed = "[" + for frame in received_down_frames: + down_frames_printed += f"{frame.__class__.__name__}, " + down_frames_printed += "]" + expected_frames_printed = "[" + for frame in expected_down_frames: + expected_frames_printed += f"{frame.__name__}, " + expected_frames_printed += "]" + print("received DOWN frames =", down_frames_printed) + print("expected DOWN frames =", expected_frames_printed) assert len(received_down_frames) == len(expected_down_frames) diff --git a/src/pipecat/utils/text/base_text_aggregator.py b/src/pipecat/utils/text/base_text_aggregator.py index 27e50fff5..07cb6c097 100644 --- a/src/pipecat/utils/text/base_text_aggregator.py +++ b/src/pipecat/utils/text/base_text_aggregator.py @@ -12,9 +12,47 @@ aggregated text should be sent for speech synthesis. """ from abc import ABC, abstractmethod +from dataclasses import dataclass +from enum import Enum from typing import Optional +class AggregationType(str, Enum): + """Built-in aggregation strings.""" + + SENTENCE = "sentence" + WORD = "word" + + def __str__(self): + return self.value + + +@dataclass +class Aggregation: + """Data class representing aggregated text and its type. + + An Aggregation object is created whenever a stream of text is aggregated by + a text aggregator. It contains the aggregated text and a type indicating + the nature of the aggregation. + + Parameters: + text: The aggregated text content. + type: The type of aggregation the text represents (e.g., 'sentence', 'word', 'token', + 'my_custom_aggregation'). + """ + + text: str + type: str + + def __str__(self) -> str: + """Return a string representation of the aggregation. + + Returns: + A descriptive string showing the type and text of the aggregation. + """ + return f"Aggregation by {self.type}: {self.text}" + + class BaseTextAggregator(ABC): """Base class for text aggregators in the Pipecat framework. @@ -30,7 +68,7 @@ class BaseTextAggregator(ABC): @property @abstractmethod - def text(self) -> str: + def text(self) -> Aggregation: """Get the currently aggregated text. Subclasses must implement this property to return the text that has @@ -42,25 +80,33 @@ class BaseTextAggregator(ABC): pass @abstractmethod - async def aggregate(self, text: str) -> Optional[str]: + async def aggregate(self, text: str) -> Optional[Aggregation]: """Aggregate the specified text with the currently accumulated text. This method should be implemented to define how the new text contributes - to the aggregation process. It returns the updated aggregated text if - it's ready to be processed, or None otherwise. + to the aggregation process. It returns the aggregated text and a string + describing how it was aggregated if it's ready to be processed, + or None otherwise. Subclasses should implement their specific logic for: - How to combine new text with existing accumulated text - When to consider the aggregated text ready for processing - What criteria determine text completion (e.g., sentence boundaries) + - When a completion occurs, the method should return an Aggregation object + containing the aggregated text and its type. The text should be stripped + of leading/trailing whitespace so that consumers can rely on a consistent + format. Args: - text: The text to be aggregated. + text: The text to be aggregated Returns: - The updated aggregated text if ready for processing, or None if more - text is needed before the aggregated content is ready. + An Aggregation object if ready for processing, or None if more + text is needed before the aggregated content is ready. If an Aggregation + object is returned, it should consist of the updated aggregated text, + stripped of leading/trailing whitespace, and a string indicating the + type of aggregation (e.g., 'sentence', 'word', 'token', 'my_custom_aggregation'). """ pass diff --git a/src/pipecat/utils/text/pattern_pair_aggregator.py b/src/pipecat/utils/text/pattern_pair_aggregator.py index ac074f2de..5c4cafd53 100644 --- a/src/pipecat/utils/text/pattern_pair_aggregator.py +++ b/src/pipecat/utils/text/pattern_pair_aggregator.py @@ -17,7 +17,7 @@ from typing import Awaitable, Callable, Optional, Tuple from loguru import logger from pipecat.utils.string import match_endofsentence -from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.text.base_text_aggregator import Aggregation, AggregationType, BaseTextAggregator class PatternMatch: @@ -75,13 +75,13 @@ class PatternPairAggregator(BaseTextAggregator): self._handlers = {} @property - def text(self) -> str: + def text(self) -> Aggregation: """Get the currently buffered text. Returns: The current text buffer content that hasn't been processed yet. """ - return self._text + return Aggregation(text=self._text.strip(), type=AggregationType.SENTENCE) def add_pattern_pair( self, pattern_id: str, start_pattern: str, end_pattern: str, remove_match: bool = True @@ -208,7 +208,7 @@ class PatternPairAggregator(BaseTextAggregator): return False - async def aggregate(self, text: str) -> Optional[str]: + async def aggregate(self, text: str) -> Optional[Aggregation]: """Aggregate text and process pattern pairs. This method adds the new text to the buffer, processes any complete pattern @@ -220,8 +220,9 @@ class PatternPairAggregator(BaseTextAggregator): text: New text to add to the buffer. Returns: - Processed text up to a sentence boundary, or None if more - text is needed to form a complete sentence or pattern. + An Aggregation object containing processed text up to a sentence boundary + and marked as SENTENCE type, or None if more text is needed to form a + complete sentence or pattern. """ # Add new text to buffer self._text += text @@ -244,7 +245,7 @@ class PatternPairAggregator(BaseTextAggregator): # Extract text up to the sentence boundary result = self._text[:eos_marker] self._text = self._text[eos_marker:] - return result + return Aggregation(text=result.strip(), type=AggregationType.SENTENCE) # No complete sentence found yet return None diff --git a/src/pipecat/utils/text/simple_text_aggregator.py b/src/pipecat/utils/text/simple_text_aggregator.py index f9eb7d83a..56eab7032 100644 --- a/src/pipecat/utils/text/simple_text_aggregator.py +++ b/src/pipecat/utils/text/simple_text_aggregator.py @@ -14,7 +14,7 @@ text processing scenarios. from typing import Optional from pipecat.utils.string import match_endofsentence -from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.text.base_text_aggregator import Aggregation, AggregationType, BaseTextAggregator class SimpleTextAggregator(BaseTextAggregator): @@ -33,15 +33,15 @@ class SimpleTextAggregator(BaseTextAggregator): self._text = "" @property - def text(self) -> str: + def text(self) -> Aggregation: """Get the currently aggregated text. Returns: The text that has been accumulated in the buffer. """ - return self._text + return Aggregation(text=self._text.strip(), type=AggregationType.SENTENCE) - async def aggregate(self, text: str) -> Optional[str]: + async def aggregate(self, text: str) -> Optional[Aggregation]: """Aggregate text and return completed sentences. Adds the new text to the buffer and checks for end-of-sentence markers. @@ -64,7 +64,9 @@ class SimpleTextAggregator(BaseTextAggregator): result = self._text[:eos_end_marker] self._text = self._text[eos_end_marker:] - return result + if result: + return Aggregation(text=result.strip(), type=AggregationType.SENTENCE) + return None async def handle_interruption(self): """Handle interruptions by clearing the text buffer. diff --git a/src/pipecat/utils/text/skip_tags_aggregator.py b/src/pipecat/utils/text/skip_tags_aggregator.py index 6f6f8455c..3c8b95aab 100644 --- a/src/pipecat/utils/text/skip_tags_aggregator.py +++ b/src/pipecat/utils/text/skip_tags_aggregator.py @@ -14,7 +14,7 @@ as a unit regardless of internal punctuation. from typing import Optional, Sequence from pipecat.utils.string import StartEndTags, match_endofsentence, parse_start_end_tags -from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.text.base_text_aggregator import Aggregation, AggregationType, BaseTextAggregator class SkipTagsAggregator(BaseTextAggregator): @@ -43,15 +43,15 @@ class SkipTagsAggregator(BaseTextAggregator): self._current_tag_index: int = 0 @property - def text(self) -> str: + def text(self) -> Aggregation: """Get the currently buffered text. Returns: The current text buffer content that hasn't been processed yet. """ - return self._text + return Aggregation(text=self._text.strip(), type=AggregationType.SENTENCE) - async def aggregate(self, text: str) -> Optional[str]: + async def aggregate(self, text: str) -> Optional[Aggregation]: """Aggregate text while respecting tag boundaries. This method adds the new text to the buffer, processes any complete @@ -63,8 +63,9 @@ class SkipTagsAggregator(BaseTextAggregator): text: New text to add to the buffer. Returns: - Processed text up to a sentence boundary (when not within tags), - or None if more text is needed to complete a sentence or close tags. + An Aggregation object containing text up to a sentence boundary and + marked as SENTENCE type or None if more text is needed to complete a + sentence or close tags. """ # Add new text to buffer self._text += text @@ -80,7 +81,7 @@ class SkipTagsAggregator(BaseTextAggregator): # Extract text up to the sentence boundary result = self._text[:eos_marker] self._text = self._text[eos_marker:] - return result + return Aggregation(text=result.strip(), type=AggregationType.SENTENCE) # No complete sentence found yet return None diff --git a/tests/test_pattern_pair_aggregator.py b/tests/test_pattern_pair_aggregator.py index 8426dcf39..01e45f9bd 100644 --- a/tests/test_pattern_pair_aggregator.py +++ b/tests/test_pattern_pair_aggregator.py @@ -30,7 +30,7 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): # First part doesn't complete the pattern result = await self.aggregator.aggregate("Hello pattern") self.assertIsNone(result) - self.assertEqual(self.aggregator.text, "Hello pattern") + self.assertEqual(self.aggregator.text.text, "Hello pattern") # Second part completes the pattern and includes an exclamation point result = await self.aggregator.aggregate(" content!") @@ -45,14 +45,16 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): # The exclamation point should be treated as a sentence boundary, # so the result should include just text up to and including "!" - self.assertEqual(result, "Hello !") + self.assertEqual(result.text, "Hello !") + self.assertEqual(result.type, "sentence") - # Next sentence should be processed separately + # Next sentence should be processed separately. Spaces around the sentence + # should be stripped in the returned Aggregation. result = await self.aggregator.aggregate(" This is another sentence.") - self.assertEqual(result, " This is another sentence.") - + self.assertEqual(result.text, "This is another sentence.") + self.assertEqual(result.type, "sentence") # Buffer should be empty after returning a complete sentence - self.assertEqual(self.aggregator.text, "") + self.assertEqual(self.aggregator.text.text, "") async def test_incomplete_pattern(self): # Add text with incomplete pattern @@ -65,11 +67,11 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): self.test_handler.assert_not_called() # Buffer should contain the incomplete text - self.assertEqual(self.aggregator.text, "Hello pattern content") + self.assertEqual(self.aggregator.text.text, "Hello pattern content") # Reset and confirm buffer is cleared await self.aggregator.reset() - self.assertEqual(self.aggregator.text, "") + self.assertEqual(self.aggregator.text.text, "") async def test_multiple_patterns(self): # Set up multiple patterns and handlers @@ -106,10 +108,11 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): self.assertEqual(emphasis_match.content, "very") # Voice pattern should be removed, emphasis pattern should remain - self.assertEqual(result, "Hello I am very excited to meet you!") + self.assertEqual(result.text, "Hello I am very excited to meet you!") + self.assertEqual(result.type, "sentence") # Buffer should be empty - self.assertEqual(self.aggregator.text, "") + self.assertEqual(self.aggregator.text.text, "") async def test_handle_interruption(self): # Start with incomplete pattern @@ -120,7 +123,7 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): await self.aggregator.handle_interruption() # Buffer should be cleared - self.assertEqual(self.aggregator.text, "") + self.assertEqual(self.aggregator.text.text, "") # Handler should not have been called self.test_handler.assert_not_called() @@ -141,7 +144,8 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): self.assertEqual(call_args.content, "This is sentence one. This is sentence two.") # Pattern should be removed, resulting in text with sentences merged - self.assertEqual(result, "Hello Final sentence.") + self.assertEqual(result.text, "Hello Final sentence.") + self.assertEqual(result.type, "sentence") # Buffer should be empty - self.assertEqual(self.aggregator.text, "") + self.assertEqual(self.aggregator.text.text, "") diff --git a/tests/test_simple_text_aggregator.py b/tests/test_simple_text_aggregator.py index ff6dd1847..f8e2ee553 100644 --- a/tests/test_simple_text_aggregator.py +++ b/tests/test_simple_text_aggregator.py @@ -15,15 +15,21 @@ class TestSimpleTextAggregator(unittest.IsolatedAsyncioTestCase): async def test_reset_aggregations(self): assert await self.aggregator.aggregate("Hello ") == None - assert self.aggregator.text == "Hello " + assert self.aggregator.text.text == "Hello" await self.aggregator.reset() - assert self.aggregator.text == "" + assert self.aggregator.text.text == "" async def test_simple_sentence(self): assert await self.aggregator.aggregate("Hello ") == None - assert await self.aggregator.aggregate("Pipecat!") == "Hello Pipecat!" - assert self.aggregator.text == "" + aggregate = await self.aggregator.aggregate("Pipecat!") + assert aggregate.text == "Hello Pipecat!" + assert aggregate.type == "sentence" + assert self.aggregator.text.text == "" async def test_multiple_sentences(self): - assert await self.aggregator.aggregate("Hello Pipecat! How are ") == "Hello Pipecat!" - assert await self.aggregator.aggregate("you?") == " How are you?" + aggregate = await self.aggregator.aggregate("Hello Pipecat! How are ") + assert aggregate.text == "Hello Pipecat!" + # Aggregators should strip leading/trailing spaces when returning text + assert self.aggregator.text.text == "How are" + aggregate = await self.aggregator.aggregate("you?") + assert aggregate.text == "How are you?" diff --git a/tests/test_skip_tags_aggregator.py b/tests/test_skip_tags_aggregator.py index f6cbb7b93..702b991ce 100644 --- a/tests/test_skip_tags_aggregator.py +++ b/tests/test_skip_tags_aggregator.py @@ -18,16 +18,18 @@ class TestSkipTagsAggregator(unittest.IsolatedAsyncioTestCase): # No tags involved, aggregate at end of sentence. result = await self.aggregator.aggregate("Hello Pipecat!") - self.assertEqual(result, "Hello Pipecat!") - self.assertEqual(self.aggregator.text, "") + self.assertEqual(result.text, "Hello Pipecat!") + self.assertEqual(result.type, "sentence") + self.assertEqual(self.aggregator.text.text, "") async def test_basic_tags(self): await self.aggregator.reset() # Tags involved, avoid aggregation during tags. result = await self.aggregator.aggregate("My email is foo@pipecat.ai.") - self.assertEqual(result, "My email is foo@pipecat.ai.") - self.assertEqual(self.aggregator.text, "") + self.assertEqual(result.text, "My email is foo@pipecat.ai.") + self.assertEqual(result.type, "sentence") + self.assertEqual(self.aggregator.text.text, "") async def test_streaming_tags(self): await self.aggregator.reset() @@ -35,20 +37,22 @@ class TestSkipTagsAggregator(unittest.IsolatedAsyncioTestCase): # Tags involved, stream small chunk of texts. result = await self.aggregator.aggregate("My email is foo.") self.assertIsNone(result) - self.assertEqual(self.aggregator.text, "My email is foo.") + self.assertEqual(self.aggregator.text.text, "My email is foo.") result = await self.aggregator.aggregate("bar@pipecat.") self.assertIsNone(result) - self.assertEqual(self.aggregator.text, "My email is foo.bar@pipecat.") + self.assertEqual(self.aggregator.text.text, "My email is foo.bar@pipecat.") result = await self.aggregator.aggregate("aifoo.bar@pipecat.aifoo.bar@pipecat.ai.") - self.assertEqual(result, "My email is foo.bar@pipecat.ai.") - self.assertEqual(self.aggregator.text, "") + self.assertEqual(result.text, "My email is foo.bar@pipecat.ai.") + self.assertEqual(self.aggregator.text.text, "") + self.assertEqual(self.aggregator.text.type, "sentence") From 24266c238ff316d25403a34ceb660766c6114d92 Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Fri, 14 Nov 2025 18:31:58 -0500 Subject: [PATCH 087/110] Augmented PatternPairAggregator so that matched patterns can... be treated as their own aggregation, taking advantage of the new ability to assign a type to an aggregation --- CHANGELOG.md | 27 ++ .../35-pattern-pair-voice-switching.py | 16 +- src/pipecat/extensions/ivr/ivr_navigator.py | 18 +- .../utils/text/pattern_pair_aggregator.py | 236 +++++++++++++----- tests/test_pattern_pair_aggregator.py | 74 ++++-- 5 files changed, 283 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 497e97f2c..5ee809beb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,33 @@ Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and produce/consume `Aggregation` objects. - All uses of the above Aggregators have been updated accordingly. +- Augmented the `PatternPairAggregator` so that matched patterns can be treated as their own + aggregation, taking advantage of the new. To that end: + - Introduced a new, preferred version of `add_pattern` to support a new option for treating a + match as a separate aggregation returned from `aggregate()`. This replaces the now + deprecated `add_pattern_pair` method and you provide a `MatchAction` in lieu of the `remove_match` field. + - `MatchAction` enum: `REMOVE`, `KEEP`, `AGGREGATE`, allowing customization for how + a match should be handled. + - `REMOVE`: The text along with its delimiters will be removed from the streaming text. + Sentence aggregation will continue on as if this text did not exist. + - `KEEP`: The delimiters will be removed, but the content between them will be kept. + Sentence aggregation will continue on with the internal text included. + - `AGGREGATE`: The delimiters will be removed and the content between will be treated + as a separate aggregation. Any text before the start of the pattern will be + returned early, whether or not a complete sentence was found. Then the pattern + will be returned. Then the aggregation will continue on sentence matching after + the closing delimiter is found. The content between the delimiters is not + aggregated by sentence. It is aggregated as one single block of text. + - `PatternMatch` now extends `Aggregation` and provides richer info to handlers. + - **BREAKING**: The `PatternMatch` type returned to handlers registered via `on_pattern_match` + has been updated to subclass from the new `Aggregation` type, which means that `content` + has been replaced with `text` and `pattern_id` has been replaced with `type`: + ``` + async dev on_match_tag(match: PatternMatch): + pattern = match.type # instead of match.pattern_id + text = match.text # instead of match.content + ``` + ### Deprecated - The `api_key` parameter in `GeminiTTSService` is deprecated. Use diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index 7ed9eb268..3a102acfd 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -62,7 +62,11 @@ from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams -from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator +from pipecat.utils.text.pattern_pair_aggregator import ( + MatchAction, + PatternMatch, + PatternPairAggregator, +) load_dotenv(override=True) @@ -106,16 +110,16 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): pattern_aggregator = PatternPairAggregator() # Add pattern for voice switching - pattern_aggregator.add_pattern_pair( - pattern_id="voice_tag", + pattern_aggregator.add_pattern( + type="voice", start_pattern="", end_pattern="", - remove_match=True, + action=MatchAction.REMOVE, # Remove tags from final text ) # Register handler for voice switching async def on_voice_tag(match: PatternMatch): - voice_name = match.content.strip().lower() + voice_name = match.text.strip().lower() if voice_name in VOICE_IDS: # First flush any existing audio to finish the current context await tts.flush_audio() @@ -125,7 +129,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): else: logger.warning(f"Unknown voice: {voice_name}") - pattern_aggregator.on_pattern_match("voice_tag", on_voice_tag) + pattern_aggregator.on_pattern_match("voice", on_voice_tag) stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) diff --git a/src/pipecat/extensions/ivr/ivr_navigator.py b/src/pipecat/extensions/ivr/ivr_navigator.py index 9c59772ad..1ddb41ed8 100644 --- a/src/pipecat/extensions/ivr/ivr_navigator.py +++ b/src/pipecat/extensions/ivr/ivr_navigator.py @@ -31,7 +31,11 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.llm_service import LLMService -from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator +from pipecat.utils.text.pattern_pair_aggregator import ( + MatchAction, + PatternMatch, + PatternPairAggregator, +) class IVRStatus(Enum): @@ -114,15 +118,15 @@ class IVRProcessor(FrameProcessor): def _setup_xml_patterns(self): """Set up XML pattern detection and handlers.""" # Register DTMF pattern - self._aggregator.add_pattern_pair("dtmf", "", "", remove_match=True) + self._aggregator.add_pattern("dtmf", "", "", action=MatchAction.REMOVE) self._aggregator.on_pattern_match("dtmf", self._handle_dtmf_action) # Register mode pattern - self._aggregator.add_pattern_pair("mode", "", "", remove_match=True) + self._aggregator.add_pattern("mode", "", "", action=MatchAction.REMOVE) self._aggregator.on_pattern_match("mode", self._handle_mode_action) # Register IVR pattern - self._aggregator.add_pattern_pair("ivr", "", "", remove_match=True) + self._aggregator.add_pattern("ivr", "", "", action=MatchAction.REMOVE) self._aggregator.on_pattern_match("ivr", self._handle_ivr_action) async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -159,7 +163,7 @@ class IVRProcessor(FrameProcessor): Args: match: The pattern match containing DTMF content. """ - value = match.content + value = match.text logger.debug(f"DTMF detected: {value}") try: @@ -180,7 +184,7 @@ class IVRProcessor(FrameProcessor): Args: match: The pattern match containing IVR status content. """ - status = match.content + status = match.text logger.trace(f"IVR status detected: {status}") # Convert string to enum, with validation @@ -211,7 +215,7 @@ class IVRProcessor(FrameProcessor): Args: match: The pattern match containing mode content. """ - mode = match.content + mode = match.text logger.debug(f"Mode detected: {mode}") if mode == "conversation": await self._handle_conversation() diff --git a/src/pipecat/utils/text/pattern_pair_aggregator.py b/src/pipecat/utils/text/pattern_pair_aggregator.py index 5c4cafd53..c140e3243 100644 --- a/src/pipecat/utils/text/pattern_pair_aggregator.py +++ b/src/pipecat/utils/text/pattern_pair_aggregator.py @@ -8,11 +8,12 @@ This module provides an aggregator that identifies and processes content between pattern pairs (like XML tags or custom delimiters) in streaming text, with -support for custom handlers and configurable pattern removal. +support for custom handlers and configurable actions for when a pattern is found. """ import re -from typing import Awaitable, Callable, Optional, Tuple +from enum import Enum +from typing import Awaitable, Callable, List, Optional, Tuple from loguru import logger @@ -20,7 +21,28 @@ from pipecat.utils.string import match_endofsentence from pipecat.utils.text.base_text_aggregator import Aggregation, AggregationType, BaseTextAggregator -class PatternMatch: +class MatchAction(Enum): + """Actions to take when a pattern pair is matched. + + Parameters: + REMOVE: The text along with its delimiters will be removed from the streaming text. + Sentence aggregation will continue on as if this text did not exist. + KEEP: The delimiters will be removed, but the content between them will be kept. + Sentence aggregation will continue on with the internal text included. + AGGREGATE: The delimiters will be removed and the content between will be treated + as a separate aggregation. Any text before the start of the pattern will be + returned early, whether or not a complete sentence was found. Then the pattern + will be returned. Then the aggregation will continue on sentence matching after + the closing delimiter is found. The content between the delimiters is not + aggregated by sentence. It is aggregated as one single block of text. + """ + + REMOVE = "remove" + KEEP = "keep" + AGGREGATE = "aggregate" + + +class PatternMatch(Aggregation): """Represents a matched pattern pair with its content. A PatternMatch object is created when a complete pattern pair is found @@ -29,25 +51,25 @@ class PatternMatch: content between the patterns. """ - def __init__(self, pattern_id: str, full_match: str, content: str): + def __init__(self, content: str, type: str, full_match: str): """Initialize a pattern match. Args: - pattern_id: The identifier of the matched pattern pair. + type: The type of the matched pattern pair. It should be representative + of the content type (e.g., 'sentence', 'code', 'speaker', 'custom'). full_match: The complete text including start and end patterns. content: The text content between the start and end patterns. """ - self.pattern_id = pattern_id + super().__init__(text=content, type=type) self.full_match = full_match - self.content = content def __str__(self) -> str: """Return a string representation of the pattern match. Returns: - A descriptive string showing the pattern ID and content. + A descriptive string showing the pattern type and content. """ - return f"PatternMatch(id={self.pattern_id}, content={self.content})" + return f"PatternMatch(type={self.type}, text={self.text}, full_match={self.full_match})" class PatternPairAggregator(BaseTextAggregator): @@ -55,16 +77,21 @@ class PatternPairAggregator(BaseTextAggregator): This aggregator buffers text until it can identify complete pattern pairs (defined by start and end patterns), processes the content between these - patterns using registered handlers, and returns text at sentence boundaries. - It's particularly useful for processing structured content in streaming text, - such as XML tags, markdown formatting, or custom delimiters. + patterns using registered handlers. By default, its aggregation method + returns text at sentence boundaries, and remove the content found between + any matched patterns. However, matched patterns can also be configured to + returned as a separate aggregation object containing the content between + their start and end patterns or left in, so that only the delimiters are + removed and a callback can be triggered. + + This aggregator is particularly useful for processing structured content in + streaming text, such as XML tags, markdown formatting, or custom delimiters. The aggregator ensures that patterns spanning multiple text chunks are - correctly identified and handles cases where patterns contain sentence - boundaries. + correctly identified. """ - def __init__(self): + def __init__(self, **kwargs): """Initialize the pattern pair aggregator. Creates an empty aggregator with no patterns or handlers registered. @@ -76,15 +103,26 @@ class PatternPairAggregator(BaseTextAggregator): @property def text(self) -> Aggregation: - """Get the currently buffered text. + """Get the currently aggregated text. Returns: - The current text buffer content that hasn't been processed yet. + The text that has been accumulated in the buffer. """ - return Aggregation(text=self._text.strip(), type=AggregationType.SENTENCE) + pattern_start = self._match_start_of_pattern(self._text) + stripped_text = self._text.strip() + type = ( + pattern_start[1].get("type", AggregationType.SENTENCE) + if pattern_start + else AggregationType.SENTENCE + ) + return Aggregation(text=stripped_text, type=type) - def add_pattern_pair( - self, pattern_id: str, start_pattern: str, end_pattern: str, remove_match: bool = True + def add_pattern( + self, + type: str, + start_pattern: str, + end_pattern: str, + action: MatchAction = MatchAction.REMOVE, ) -> "PatternPairAggregator": """Add a pattern pair to detect in the text. @@ -93,41 +131,94 @@ class PatternPairAggregator(BaseTextAggregator): the end pattern, and treat the content between them as a match. Args: - pattern_id: Unique identifier for this pattern pair. + type: Identifier for this pattern pair. Should be unique and ideally descriptive. + (e.g., 'code', 'speaker', 'custom'). type can not be 'sentence' or 'word' as + those are reserved for the default behavior. start_pattern: Pattern that marks the beginning of content. end_pattern: Pattern that marks the end of content. - remove_match: Whether to remove the matched content from the text. + action: What to do when a complete pattern is matched: + - MatchAction.REMOVE: Remove the matched pattern from the text. + - MatchAction.KEEP: Keep the matched pattern in the text and treat it as + normal text. This allows you to register handlers for + the pattern without affecting the aggregation logic. + - MatchAction.AGGREGATE: Return the matched pattern as a separate + aggregation object. Returns: Self for method chaining. """ - self._patterns[pattern_id] = { + if type in [AggregationType.SENTENCE, AggregationType.WORD]: + raise ValueError( + f"The aggregation type '{type}' is reserved for default behavior and can not be used for custom patterns." + ) + self._patterns[type] = { "start": start_pattern, "end": end_pattern, - "remove_match": remove_match, + "type": type, + "action": action, } return self + def add_pattern_pair( + self, pattern_id: str, start_pattern: str, end_pattern: str, remove_match: bool = True + ): + """Add a pattern pair to detect in the text. + + .. deprecated:: 0.0.95 + This function is deprecated and will be removed in a future version. + Use `add_pattern` with a type and MatchAction instead. + + This method calls `add_pattern` setting type with the provided pattern_id and action + to either MatchAction.REMOVE or MatchAction.KEEP based on `remove_match`. + + Args: + pattern_id: Identifier for this pattern pair. Should be unique and ideally descriptive. + (e.g., 'code', 'speaker', 'custom'). pattern_id can not be 'sentence' or 'word' + as those arereserved for the default behavior. + start_pattern: Pattern that marks the beginning of content. + end_pattern: Pattern that marks the end of content. + remove_match: If True, the matched pattern will be removed from the text. (Same as MatchAction.REMOVE) + If False, it will be kept and treated as normal text. (Same as MatchAction.KEEP) + """ + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("once") + warnings.warn( + "add_pattern_pair with a pattern_id or remove_match is deprecated and will be" + " removed in a future version. Use add_pattern with a type and MatchAction instead", + DeprecationWarning, + stacklevel=2, + ) + + action = MatchAction.REMOVE if remove_match else MatchAction.KEEP + return self.add_pattern( + type=pattern_id, + start_pattern=start_pattern, + end_pattern=end_pattern, + action=action, + ) + def on_pattern_match( - self, pattern_id: str, handler: Callable[[PatternMatch], Awaitable[None]] + self, type: str, handler: Callable[[PatternMatch], Awaitable[None]] ) -> "PatternPairAggregator": """Register a handler for when a pattern pair is matched. The handler will be called whenever a complete match for the - specified pattern ID is found in the text. + specified type is found in the text. Args: - pattern_id: ID of the pattern pair to match. + type: The type of the pattern pair to trigger the handler. handler: Async function to call when pattern is matched. The function should accept a PatternMatch object. Returns: Self for method chaining. """ - self._handlers[pattern_id] = handler + self._handlers[type] = handler return self - async def _process_complete_patterns(self, text: str) -> Tuple[str, bool]: + async def _process_complete_patterns(self, text: str) -> Tuple[List[PatternMatch], str]: """Process all complete pattern pairs in the text. Searches for all complete pattern pairs in the text, calls the @@ -137,19 +228,19 @@ class PatternPairAggregator(BaseTextAggregator): text: The text to process for pattern matches. Returns: - Tuple of (processed_text, was_modified) where: + Tuple of (all_matches, processed_text) where: - - processed_text is the text after processing patterns - - was_modified indicates whether any changes were made + - all_matches is a list of all pattern matches found. Note: There really should only ever be 1. + - processed_text is the text after processing patterns. If no patterns are found, it will be the same as input text. """ + all_matches = [] processed_text = text - modified = False - for pattern_id, pattern_info in self._patterns.items(): + for type, pattern_info in self._patterns.items(): # Escape special regex characters in the patterns start = re.escape(pattern_info["start"]) end = re.escape(pattern_info["end"]) - remove_match = pattern_info["remove_match"] + action = pattern_info["action"] # Create regex to match from start pattern to end pattern # The .*? is non-greedy to handle nested patterns @@ -165,24 +256,25 @@ class PatternPairAggregator(BaseTextAggregator): # Create pattern match object pattern_match = PatternMatch( - pattern_id=pattern_id, full_match=full_match, content=content + content=content.strip(), type=type, full_match=full_match ) # Call the appropriate handler if registered - if pattern_id in self._handlers: + if type in self._handlers: try: - await self._handlers[pattern_id](pattern_match) + await self._handlers[type](pattern_match) except Exception as e: - logger.error(f"Error in pattern handler for {pattern_id}: {e}") + logger.error(f"Error in pattern handler for {type}: {e}") # Remove the pattern from the text if configured - if remove_match: + if action == MatchAction.REMOVE: processed_text = processed_text.replace(full_match, "", 1) - modified = True + else: + all_matches.append(pattern_match) - return processed_text, modified + return all_matches, processed_text - def _has_incomplete_patterns(self, text: str) -> bool: + def _match_start_of_pattern(self, text: str) -> Optional[Tuple[int, dict]]: """Check if text contains incomplete pattern pairs. Determines whether the text contains any start patterns without @@ -192,9 +284,10 @@ class PatternPairAggregator(BaseTextAggregator): text: The text to check for incomplete patterns. Returns: - True if there are incomplete patterns, False otherwise. + A tuple of (start_index, pattern_info) if an incomplete pattern is found, + or None if no patterns are found or all patterns are complete. """ - for pattern_id, pattern_info in self._patterns.items(): + for type, pattern_info in self._patterns.items(): start = pattern_info["start"] end = pattern_info["end"] @@ -203,12 +296,16 @@ class PatternPairAggregator(BaseTextAggregator): end_count = text.count(end) # If there are more starts than ends, we have incomplete patterns + # Again, this is written generically but there only ever should + # be one pattern active at a time, so the counts should be 0 or 1. + # Which is why we base the return on the first found. if start_count > end_count: - return True + start_index = text.find(start) + return [start_index, pattern_info] - return False + return None - async def aggregate(self, text: str) -> Optional[Aggregation]: + async def aggregate(self, text: str) -> Optional[PatternMatch]: """Aggregate text and process pattern pairs. This method adds the new text to the buffer, processes any complete pattern @@ -220,24 +317,43 @@ class PatternPairAggregator(BaseTextAggregator): text: New text to add to the buffer. Returns: - An Aggregation object containing processed text up to a sentence boundary - and marked as SENTENCE type, or None if more text is needed to form a - complete sentence or pattern. + Processed text up to a sentence boundary, or None if more + text is needed to form a complete sentence or pattern. """ # Add new text to buffer self._text += text # Process any complete patterns in the buffer - processed_text, modified = await self._process_complete_patterns(self._text) + patterns, processed_text = await self._process_complete_patterns(self._text) - # Only update the buffer if modifications were made - if modified: - self._text = processed_text + self._text = processed_text + + if len(patterns) > 0: + if len(patterns) > 1: + logger.warning( + f"Multiple patterns matched: {[p.type for p in patterns]}. Only the first pattern will be returned." + ) + # If the pattern found is set to be aggregated, return it + action = self._patterns[patterns[0].type].get("action", MatchAction.REMOVE) + if action == MatchAction.AGGREGATE: + self._text = "" + return patterns[0] # Check if we have incomplete patterns - if self._has_incomplete_patterns(self._text): - # Still waiting for complete patterns - return None + pattern_start = self._match_start_of_pattern(self._text) + if pattern_start is not None: + # If the start pattern is at the beginning or should not be separately aggregated, return None + if ( + pattern_start[0] == 0 + or pattern_start[1].get("action", MatchAction.REMOVE) != MatchAction.AGGREGATE + ): + return None + # Otherwise, strip the text up to the start pattern and return it + result = self._text[: pattern_start[0]] + self._text = self._text[pattern_start[0] :] + return PatternMatch( + content=result.strip(), type=AggregationType.SENTENCE, full_match=result + ) # Find sentence boundary if no incomplete patterns eos_marker = match_endofsentence(self._text) @@ -245,7 +361,9 @@ class PatternPairAggregator(BaseTextAggregator): # Extract text up to the sentence boundary result = self._text[:eos_marker] self._text = self._text[eos_marker:] - return Aggregation(text=result.strip(), type=AggregationType.SENTENCE) + return PatternMatch( + content=result.strip(), type=AggregationType.SENTENCE, full_match=result + ) # No complete sentence found yet return None diff --git a/tests/test_pattern_pair_aggregator.py b/tests/test_pattern_pair_aggregator.py index 01e45f9bd..20b44a03c 100644 --- a/tests/test_pattern_pair_aggregator.py +++ b/tests/test_pattern_pair_aggregator.py @@ -7,30 +7,42 @@ import unittest from unittest.mock import AsyncMock -from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator +from pipecat.utils.text.pattern_pair_aggregator import ( + MatchAction, + PatternMatch, + PatternPairAggregator, +) class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): def setUp(self): self.aggregator = PatternPairAggregator() self.test_handler = AsyncMock() + self.code_handler = AsyncMock() # Add a test pattern self.aggregator.add_pattern_pair( pattern_id="test_pattern", start_pattern="", end_pattern="", - remove_match=True, + ) + self.aggregator.add_pattern( + type="code_pattern", + start_pattern="", + end_pattern="", + action=MatchAction.AGGREGATE, ) # Register the mock handler self.aggregator.on_pattern_match("test_pattern", self.test_handler) + self.aggregator.on_pattern_match("code_pattern", self.code_handler) async def test_pattern_match_and_removal(self): # First part doesn't complete the pattern result = await self.aggregator.aggregate("Hello pattern") self.assertIsNone(result) self.assertEqual(self.aggregator.text.text, "Hello pattern") + self.assertEqual(self.aggregator.text.type, "test_pattern") # Second part completes the pattern and includes an exclamation point result = await self.aggregator.aggregate(" content!") @@ -39,9 +51,9 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): self.test_handler.assert_called_once() call_args = self.test_handler.call_args[0][0] self.assertIsInstance(call_args, PatternMatch) - self.assertEqual(call_args.pattern_id, "test_pattern") + self.assertEqual(call_args.type, "test_pattern") self.assertEqual(call_args.full_match, "pattern content") - self.assertEqual(call_args.content, "pattern content") + self.assertEqual(call_args.text, "pattern content") # The exclamation point should be treated as a sentence boundary, # so the result should include just text up to and including "!" @@ -52,7 +64,35 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): # should be stripped in the returned Aggregation. result = await self.aggregator.aggregate(" This is another sentence.") self.assertEqual(result.text, "This is another sentence.") + + # Buffer should be empty after returning a complete sentence + self.assertEqual(self.aggregator.text.text, "") + + async def test_pattern_match_and_aggregate(self): + # First part doesn't complete the pattern + result = await self.aggregator.aggregate("Here is code pattern") + self.assertEqual(result.text, "Here is code") + self.assertEqual(self.aggregator.text.text, "pattern") + self.assertEqual(self.aggregator.text.type, "code_pattern") + + # Second part completes the pattern and includes an exclamation point + result = await self.aggregator.aggregate(" content") + + # Verify the handler was called with correct PatternMatch object + self.code_handler.assert_called_once() + call_args = self.code_handler.call_args[0][0] + self.assertIsInstance(call_args, PatternMatch) + self.assertEqual(call_args.type, "code_pattern") + self.assertEqual(call_args.full_match, "pattern content") + self.assertEqual(call_args.text, "pattern content") + self.assertEqual(result.text, "pattern content") + self.assertEqual(result.type, "code_pattern") + + # Next sentence should be processed separately + result = await self.aggregator.aggregate(" This is another sentence.") + self.assertEqual(result.text, "This is another sentence.") self.assertEqual(result.type, "sentence") + # Buffer should be empty after returning a complete sentence self.assertEqual(self.aggregator.text.text, "") @@ -68,6 +108,7 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): # Buffer should contain the incomplete text self.assertEqual(self.aggregator.text.text, "Hello pattern content") + self.assertEqual(self.aggregator.text.type, "test_pattern") # Reset and confirm buffer is cleared await self.aggregator.reset() @@ -78,15 +119,18 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): voice_handler = AsyncMock() emphasis_handler = AsyncMock() - self.aggregator.add_pattern_pair( - pattern_id="voice", start_pattern="", end_pattern="", remove_match=True + self.aggregator.add_pattern( + type="voice", + start_pattern="", + end_pattern="", + action=MatchAction.REMOVE, ) - self.aggregator.add_pattern_pair( - pattern_id="emphasis", + self.aggregator.add_pattern( + type="emphasis", start_pattern="", end_pattern="", - remove_match=False, # Keep emphasis tags + action=MatchAction.KEEP, # Keep emphasis tags ) self.aggregator.on_pattern_match("voice", voice_handler) @@ -99,17 +143,16 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): # Both handlers should be called with correct data voice_handler.assert_called_once() voice_match = voice_handler.call_args[0][0] - self.assertEqual(voice_match.pattern_id, "voice") - self.assertEqual(voice_match.content, "female") + self.assertEqual(voice_match.type, "voice") + self.assertEqual(voice_match.text, "female") emphasis_handler.assert_called_once() emphasis_match = emphasis_handler.call_args[0][0] - self.assertEqual(emphasis_match.pattern_id, "emphasis") - self.assertEqual(emphasis_match.content, "very") + self.assertEqual(emphasis_match.type, "emphasis") + self.assertEqual(emphasis_match.text, "very") # Voice pattern should be removed, emphasis pattern should remain self.assertEqual(result.text, "Hello I am very excited to meet you!") - self.assertEqual(result.type, "sentence") # Buffer should be empty self.assertEqual(self.aggregator.text.text, "") @@ -141,11 +184,10 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): # Handler should be called with entire content self.test_handler.assert_called_once() call_args = self.test_handler.call_args[0][0] - self.assertEqual(call_args.content, "This is sentence one. This is sentence two.") + self.assertEqual(call_args.text, "This is sentence one. This is sentence two.") # Pattern should be removed, resulting in text with sentences merged self.assertEqual(result.text, "Hello Final sentence.") - self.assertEqual(result.type, "sentence") # Buffer should be empty self.assertEqual(self.aggregator.text.text, "") From 0e820a01b90a5531ca0377f4eb733f1a82f3e29a Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Mon, 17 Nov 2025 12:12:48 -0500 Subject: [PATCH 088/110] Introduce `append_to_context` to `TextFrame`s Adding support for setting whether or not the text in the TextFrame should be added to the LLM context (by the LLM assistant aggregator). Defaults to `True`. --- CHANGELOG.md | 4 ++++ src/pipecat/frames/frames.py | 3 +++ src/pipecat/processors/aggregators/llm_response.py | 2 +- src/pipecat/processors/aggregators/llm_response_universal.py | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ee809beb..480bfa771 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,10 @@ Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and text = match.text # instead of match.content ``` + - `TextFrame` now includes the field `append_to_context` to support setting whether or not the + encompassing text should be added to the LLM context (by the LLM assistant aggregator). It + defaults to `True`. + ### Deprecated - The `api_key` parameter in `GeminiTTSService` is deprecated. Use diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index ddb4e5a14..d703706cf 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -337,11 +337,14 @@ class TextFrame(DataFrame): # mandatory fields of theirs to have defaults to preserve # non-default-before-default argument order) includes_inter_frame_spaces: bool = field(init=False) + # Whether this text frame should be appended to the LLM context. + append_to_context: bool = field(init=False) def __post_init__(self): super().__post_init__() self.skip_tts = False self.includes_inter_frame_spaces = False + self.append_to_context = True def __str__(self): pts = format_pts(self.pts) diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index afc091d5a..ec13b643f 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -1001,7 +1001,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): await self.push_aggregation() async def _handle_text(self, frame: TextFrame): - if not self._started: + if not self._started or not frame.append_to_context: return if self._params.expect_stripped_words: diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 87974bed2..69fc649ce 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -811,7 +811,7 @@ class LLMAssistantAggregator(LLMContextAggregator): await self.push_aggregation() async def _handle_text(self, frame: TextFrame): - if not self._started: + if not self._started or not frame.append_to_context: return # Make sure we really have text (spaces count, too!) From 7a4372a9091bde4b2158661782282760faf24448 Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Mon, 17 Nov 2025 14:55:04 -0500 Subject: [PATCH 089/110] Introduced a new AggregatedTextFrame Frame type that TTSTextFrame inherits from This frame introduces an `aggregated_by` field to describe the type of text included in the frame and allows unspoken groupings of text to be pushed through the pipeline and treated similar to TTSTextFrames. --- CHANGELOG.md | 12 ++++++ src/pipecat/frames/frames.py | 27 ++++++++++++- src/pipecat/services/aws/nova_sonic/llm.py | 7 +++- .../services/google/gemini_live/llm.py | 3 +- src/pipecat/services/openai/realtime/llm.py | 3 +- .../services/openai_realtime_beta/openai.py | 3 +- src/pipecat/services/tts_service.py | 28 ++++++++----- tests/test_transcript_processor.py | 39 ++++++++++--------- 8 files changed, 88 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 480bfa771..7289ff4c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added word-level timestamps support to Hume TTS service +- Introduced a new `AggregatedTextFrame` type to support passing text along with an + `aggregated_by` field to describe the type of text included. `TTSTextFrame`s now + inherit from `AggregatedTextFrame`. With this inheritance, an observer can watch for + `AggregatedTextFrame`s to accumlate the perceived output and determine whether or not + the text was spoken based on if that frame is also a `TTSTextFrame`. + + With this frame, the llm token stream can be transformed into custom composable + chunks, allowing for aggregation outside the TTS service. This makes it possible to + listen for or handle those aggregations and sets the stage for doing things like + composing a best effort of the perceived llm output in a more digestable form and + to do so whether or not it is processed by a TTS or if even a TTS exists. + ### Changed - ⚠️ Breaking change: `LLMContext.create_image_message()`, diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index d703706cf..e437d48a1 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -12,6 +12,7 @@ and LLM processing. """ from dataclasses import dataclass, field +from enum import Enum from typing import ( TYPE_CHECKING, Any, @@ -361,8 +362,32 @@ class LLMTextFrame(TextFrame): self.includes_inter_frame_spaces = True +class AggregationType(str, Enum): + """Built-in aggregation strings.""" + + SENTENCE = "sentence" + WORD = "word" + + def __str__(self): + return self.value + + @dataclass -class TTSTextFrame(TextFrame): +class AggregatedTextFrame(TextFrame): + """Text frame representing an aggregation of TextFrames. + + This frame contains multiple TextFrames aggregated together for processing + or output along with a field to indicate how they are aggregated. + + Parameters: + aggregated_by: Method used to aggregate the text frames. + """ + + aggregated_by: AggregationType | str + + +@dataclass +class TTSTextFrame(AggregatedTextFrame): """Text frame generated by Text-to-Speech services.""" pass diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 2572b03cb..95240f748 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -27,6 +27,7 @@ from pydantic import BaseModel, Field from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter, Role from pipecat.frames.frames import ( + AggregationType, BotStoppedSpeakingFrame, CancelFrame, EndFrame, @@ -1027,7 +1028,7 @@ class AWSNovaSonicLLMService(LLMService): logger.debug(f"Assistant response text added: {text}") # Report the text of the assistant response. - frame = TTSTextFrame(text) + frame = TTSTextFrame(text, aggregated_by=AggregationType.SENTENCE) frame.includes_inter_frame_spaces = True await self.push_frame(frame) @@ -1062,7 +1063,9 @@ class AWSNovaSonicLLMService(LLMService): # TTSTextFrame would be ignored otherwise (the interruption frame # would have cleared the assistant aggregator state). await self.push_frame(LLMFullResponseStartFrame()) - frame = TTSTextFrame(self._assistant_text_buffer) + frame = TTSTextFrame( + self._assistant_text_buffer, aggregated_by=AggregationType.SENTENCE + ) frame.includes_inter_frame_spaces = True await self.push_frame(frame) self._may_need_repush_assistant_text = False diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 7e0b0f494..2c6e8e463 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -27,6 +27,7 @@ from pydantic import BaseModel, Field from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter from pipecat.frames.frames import ( + AggregationType, BotStartedSpeakingFrame, BotStoppedSpeakingFrame, CancelFrame, @@ -1644,7 +1645,7 @@ class GeminiLiveLLMService(LLMService): await self.push_frame(TTSStartedFrame()) await self.push_frame(LLMFullResponseStartFrame()) - frame = TTSTextFrame(text=text) + frame = TTSTextFrame(text=text, aggregated_by=AggregationType.SENTENCE) # Gemini Live text already includes any necessary inter-chunk spaces frame.includes_inter_frame_spaces = True diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 8eaa3d6fa..755e64040 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -19,6 +19,7 @@ from pipecat.adapters.services.open_ai_realtime_adapter import ( OpenAIRealtimeLLMAdapter, ) from pipecat.frames.frames import ( + AggregationType, BotStoppedSpeakingFrame, CancelFrame, EndFrame, @@ -684,7 +685,7 @@ class OpenAIRealtimeLLMService(LLMService): # We receive audio transcript deltas (as opposed to text deltas) when # the output modality is "audio" (the default) if evt.delta: - frame = TTSTextFrame(evt.delta) + frame = TTSTextFrame(evt.delta, aggregated_by=AggregationType.SENTENCE) # OpenAI Realtime text already includes any necessary inter-chunk spaces frame.includes_inter_frame_spaces = True await self.push_frame(frame) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index af0600882..d0cb39bf6 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -17,6 +17,7 @@ from loguru import logger from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter from pipecat.frames.frames import ( + AggregationType, BotStoppedSpeakingFrame, CancelFrame, EndFrame, @@ -652,7 +653,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_evt_audio_transcript_delta(self, evt): if evt.delta: await self.push_frame(LLMTextFrame(evt.delta)) - await self.push_frame(TTSTextFrame(evt.delta)) + await self.push_frame(TTSTextFrame(evt.delta, aggregated_by=AggregationType.SENTENCE)) async def _handle_evt_speech_started(self, evt): await self._truncate_current_audio_response() diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 33cf7d103..a4f8933ef 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -23,6 +23,8 @@ from typing import ( from loguru import logger from pipecat.frames.frames import ( + AggregatedTextFrame, + AggregationType, BotStartedSpeakingFrame, BotStoppedSpeakingFrame, CancelFrame, @@ -358,7 +360,9 @@ class TTSService(AIService): self._processing_text = False if pending_aggregation.text: - await self._push_tts_frames(pending_aggregation.text) + await self._push_tts_frames( + AggregatedTextFrame(pending_aggregation.text, pending_aggregation.type) + ) if isinstance(frame, LLMFullResponseEndFrame): if self._push_text_frames: await self.push_frame(frame, direction) @@ -368,7 +372,7 @@ class TTSService(AIService): # Store if we were processing text or not so we can set it back. processing_text = self._processing_text # Assumption: text in TTSSpeakFrame does not include inter-frame spaces - await self._push_tts_frames(frame.text) + await self._push_tts_frames(AggregatedTextFrame(frame.text, AggregationType.SENTENCE)) # We pause processing incoming frames because we are sending data to # the TTS. We pause to avoid audio overlapping. await self._maybe_pause_frame_processing() @@ -462,18 +466,24 @@ class TTSService(AIService): if not self._aggregate_sentences: text = frame.text includes_inter_frame_spaces = frame.includes_inter_frame_spaces + aggregated_by = "token" else: - aggregation = await self._text_aggregator.aggregate(frame.text) - text = aggregation.text + aggregate = await self._text_aggregator.aggregate(frame.text) + if aggregate: + text = aggregate.text + aggregated_by = aggregate.type if text: - await self._push_tts_frames(text, includes_inter_frame_spaces) + logger.trace(f"Pushing TTS frames for text: {text}, {aggregated_by}") + await self._push_tts_frames( + AggregatedTextFrame(text, aggregated_by), includes_inter_frame_spaces + ) async def _push_tts_frames( - self, text: str, includes_inter_frame_spaces: Optional[bool] = False + self, src_frame: AggregatedTextFrame, includes_inter_frame_spaces: Optional[bool] = False ): # Remove leading newlines only - text = text.lstrip("\n") + text = src_frame.text.lstrip("\n") # Don't send only whitespace. This causes problems for some TTS models. But also don't # strip all whitespace, as whitespace can influence prosody. @@ -500,7 +510,7 @@ class TTSService(AIService): if self._push_text_frames: # We send the original text after the audio. This way, if we are # interrupted, the text is not added to the assistant context. - frame = TTSTextFrame(text) + frame = TTSTextFrame(text, aggregated_by=src_frame.aggregated_by) frame.includes_inter_frame_spaces = includes_inter_frame_spaces await self.push_frame(frame) @@ -630,7 +640,7 @@ class WordTTSService(TTSService): else: # Assumption: word-by-word text frames don't include spaces, so # we can rely on the default includes_inter_frame_spaces=False - frame = TTSTextFrame(word) + frame = TTSTextFrame(word, aggregated_by=AggregationType.WORD) frame.pts = self._initial_word_timestamp + timestamp if frame: last_pts = frame.pts diff --git a/tests/test_transcript_processor.py b/tests/test_transcript_processor.py index 19366086c..d86e42101 100644 --- a/tests/test_transcript_processor.py +++ b/tests/test_transcript_processor.py @@ -11,6 +11,7 @@ from datetime import datetime, timezone from typing import List, Tuple, cast from pipecat.frames.frames import ( + AggregationType, BotStartedSpeakingFrame, BotStoppedSpeakingFrame, CancelFrame, @@ -130,11 +131,11 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), # Wait for StartedSpeaking to process - TTSTextFrame(text="Hello"), - TTSTextFrame(text="world!"), - TTSTextFrame(text="How"), - TTSTextFrame(text="are"), - TTSTextFrame(text="you?"), + TTSTextFrame(text="Hello", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="world!", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="How", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="are", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="you?", aggregated_by=AggregationType.WORD), SleepFrame(), # Wait for text frames to queue BotStoppedSpeakingFrame(), ] @@ -195,9 +196,9 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), - TTSTextFrame(text=""), # Empty text - TTSTextFrame(text=" "), # Just whitespace - TTSTextFrame(text="\n"), # Just newline + TTSTextFrame(text="", aggregated_by=AggregationType.WORD), # Empty text + TTSTextFrame(text=" ", aggregated_by=AggregationType.WORD), # Just whitespace + TTSTextFrame(text="\n", aggregated_by=AggregationType.WORD), # Just newline BotStoppedSpeakingFrame(), # Pipeline ends here; run_test will automatically send EndFrame ] @@ -235,14 +236,14 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), - TTSTextFrame(text="Hello"), - TTSTextFrame(text="world!"), + TTSTextFrame(text="Hello", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="world!", aggregated_by=AggregationType.WORD), SleepFrame(), InterruptionFrame(), # User interrupts here SleepFrame(), BotStartedSpeakingFrame(), - TTSTextFrame(text="New"), - TTSTextFrame(text="response"), + TTSTextFrame(text="New", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="response", aggregated_by=AggregationType.WORD), SleepFrame(), BotStoppedSpeakingFrame(), ] @@ -299,8 +300,8 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), - TTSTextFrame(text="Hello"), - TTSTextFrame(text="world"), + TTSTextFrame(text="Hello", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="world", aggregated_by=AggregationType.WORD), # Pipeline ends here; run_test will automatically send EndFrame ] @@ -338,8 +339,8 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), - TTSTextFrame(text="Hello"), - TTSTextFrame(text="world"), + TTSTextFrame(text="Hello", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="world", aggregated_by=AggregationType.WORD), SleepFrame(), # Ensure messages are processed CancelFrame(), ] @@ -401,8 +402,8 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), SleepFrame(), - TTSTextFrame(text="Assistant"), - TTSTextFrame(text="message"), + TTSTextFrame(text="Assistant", aggregated_by=AggregationType.WORD), + TTSTextFrame(text="message", aggregated_by=AggregationType.WORD), BotStoppedSpeakingFrame(), ] @@ -439,7 +440,7 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): # Test the specific pattern shared def make_tts_text_frame(text: str) -> TTSTextFrame: - frame = TTSTextFrame(text=text) + frame = TTSTextFrame(text=text, aggregated_by=AggregationType.WORD) frame.includes_inter_frame_spaces = True return frame From 6b6d760cf1421fbbd901235defa57cc26d7f564f Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Mon, 17 Nov 2025 16:34:07 -0500 Subject: [PATCH 090/110] Introduced LLMTextProcessor and deprecatd custom text_aggregators in TTS Introduced `LLMTextProcessor`: A new processor meant to allow customization for how LLMTextFrames should be aggregated and considered. It's purpose is to turn `LLMTextFrame`s into `AggregatedTextFrame`s. By default, a TTSService will still aggregate `LLMTextFrame`s by sentence for the service to consume. However, if you wish to override how the llm text is aggregated, you should no longer override the TTS's internal text_aggregator, but instead, insert this processor between your LLM and TTS in the pipeline. --- CHANGELOG.md | 13 +++ .../aggregators/llm_text_processor.py | 106 ++++++++++++++++++ src/pipecat/services/tts_service.py | 48 ++++++-- tests/test_piper_tts.py | 4 +- 4 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 src/pipecat/processors/aggregators/llm_text_processor.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7289ff4c5..95cc4d564 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 composing a best effort of the perceived llm output in a more digestable form and to do so whether or not it is processed by a TTS or if even a TTS exists. +- Introduced `LLMTextProcessor`: A new processor meant to allow customization for how + LLMTextFrames should be aggregated and considered. It's purpose is to turn + `LLMTextFrame`s into `AggregatedTextFrame`s. By default, a TTSService will still + aggregate `LLMTextFrame`s by sentence for the service to consume. However, if you + wish to override how the llm text is aggregated, you should no longer override the + TTS's internal text_aggregator, but instead, insert this processor between your LLM + and TTS in the pipeline. + ### Changed - ⚠️ Breaking change: `LLMContext.create_image_message()`, @@ -159,6 +167,11 @@ Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and - `english_normalization` input parameter for `MiniMaxHttpTTSService` is deprecated, use `test_normalization` instead. +- The TTS constructor field, `text_aggregator` is deprecated in favor of the new + `LLMTextProcessor`. TTSServices still have an internal aggregator for support of default + behavior, but if you want to override the aggregation behavior, you should use the new + processor. + ### Fixed - Fixed a `SimliVideoService` connection issue. diff --git a/src/pipecat/processors/aggregators/llm_text_processor.py b/src/pipecat/processors/aggregators/llm_text_processor.py new file mode 100644 index 000000000..44a8dc24e --- /dev/null +++ b/src/pipecat/processors/aggregators/llm_text_processor.py @@ -0,0 +1,106 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""LLM text processor module for processing and aggregating raw LLM output text. + +This processor will convert LLMTextFrames into AggregatedTextFrames based on the +configured text aggregator. Using the customizable aggregator, it provides +functionality to handle or manipulate LLM text frames before they are sent to other +components such as TTS services or context aggregators. It can be used to pre-aggregate +and categorize, modify, or filter direct output tokens from the LLM. +""" + +from typing import Optional + +from pipecat.frames.frames import ( + AggregatedTextFrame, + EndFrame, + Frame, + InterruptionFrame, + LLMFullResponseEndFrame, + LLMTextFrame, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator + + +class LLMTextProcessor(FrameProcessor): + """A processor for handling or manipulating LLM text frames before they are processed further. + + This processor will convert LLMTextFrames into AggregatedTextFrames based on the configured + text aggregator. Using the customizable aggregator, it provides functionality to handle or + manipulate LLM text frames before they are sent to other components such as TTS services or + context aggregators. It can be used to pre-aggregate and categorize, modify, or filter direct + output tokens from the LLM. + """ + + def __init__(self, *, text_aggregator: Optional[BaseTextAggregator] = None, **kwargs): + """Initialize the LLM text processor. + + Args: + text_aggregator: An optional text aggregator to use for processing LLM text frames. By + default, a SimpleTextAggregator aggregating by sentence will be used. + **kwargs: Additional arguments passed to parent class. + + TODO: Allow transformations per aggregation type or all (and deprecate the TTS filters). + """ + super().__init__(**kwargs) + self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process an LLMTextFrames using the aggregator to generate AggregatedTextFrames. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, InterruptionFrame): + await self._handle_interruption(frame) + await self.push_frame(frame, direction) + elif isinstance(frame, LLMTextFrame): + await self._handle_llm_text(frame) + elif isinstance(frame, LLMFullResponseEndFrame): + await self._handle_llm_end(frame.skip_tts) + await self.push_frame(frame, direction) + elif isinstance(frame, EndFrame): + await self._handle_llm_end() + await self.push_frame(frame, direction) + else: + await self.push_frame(frame, direction) + + async def _handle_interruption(self, _): + """Handle interruptions by resetting the text aggregator.""" + await self._text_aggregator.handle_interruption() + + async def reset(self): + """Reset the internal state of the text processor and its aggregator.""" + await self._text_aggregator.reset() + + async def _handle_llm_text(self, in_frame: LLMTextFrame): + aggregation = await self._text_aggregator.aggregate(in_frame.text) + if aggregation: + out_frame = AggregatedTextFrame( + text=aggregation.text, + aggregated_by=aggregation.type, + ) + out_frame.skip_tts = in_frame.skip_tts + await self.push_frame(out_frame) + + async def _handle_llm_end(self, skip_tts: bool = False): + # Flush any remaining aggregated text at the end of the LLM response + aggregation = self._text_aggregator.text + await self._text_aggregator.reset() + text = aggregation.text.strip() + if text: + out_frame = AggregatedTextFrame( + text=text, + aggregated_by=aggregation.type, + ) + out_frame.skip_tts = skip_tts + await self.push_frame(out_frame) diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index a4f8933ef..2267ee29f 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -122,6 +122,10 @@ class TTSService(AIService): pause_frame_processing: Whether to pause frame processing during audio generation. sample_rate: Output sample rate for generated audio. text_aggregator: Custom text aggregator for processing incoming text. + + .. deprecated:: 0.0.95 + Use an LLMTextProcessor before the TTSService for custom text aggregation. + text_filters: Sequence of text filters to apply after aggregation. text_filter: Single text filter (deprecated, use text_filters). @@ -144,6 +148,16 @@ class TTSService(AIService): self._voice_id: str = "" self._settings: Dict[str, Any] = {} self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() + if text_aggregator: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Parameter 'text_aggregator' is deprecated. Use an LLMTextProcessor before the TTSService for custom text aggregation.", + DeprecationWarning, + ) + self._text_filters: Sequence[BaseTextFilter] = text_filters or [] self._transport_destination: Optional[str] = transport_destination self._tracing_enabled: bool = False @@ -338,6 +352,8 @@ class TTSService(AIService): and frame.skip_tts ): await self.push_frame(frame, direction) + elif isinstance(frame, AggregatedTextFrame): + await self._push_tts_frames(frame) elif ( isinstance(frame, TextFrame) and not isinstance(frame, InterimTranscriptionFrame) @@ -482,8 +498,11 @@ class TTSService(AIService): async def _push_tts_frames( self, src_frame: AggregatedTextFrame, includes_inter_frame_spaces: Optional[bool] = False ): + type = src_frame.aggregated_by + text = src_frame.text + # Remove leading newlines only - text = src_frame.text.lstrip("\n") + text = text.lstrip("\n") # Don't send only whitespace. This causes problems for some TTS models. But also don't # strip all whitespace, as whitespace can influence prosody. @@ -497,20 +516,35 @@ class TTSService(AIService): await self.start_processing_metrics() - # Process all filter. + # Process all filters. for filter in self._text_filters: await filter.reset_interruption() text = await filter.filter(text) - if text: - await self.process_generator(self.run_tts(text)) + if not text.strip(): + await self.stop_processing_metrics() + return + + # To support use cases that may want to know the text before it's spoken, we + # push the AggregatedTextFrame version before transforming and sending to TTS. + # However, we do not want to add this text to the assistant context until it + # is spoken, so we set append_to_context to False. + src_frame.append_to_context = False + await self.push_frame(src_frame) + + await self.process_generator(self.run_tts(text)) await self.stop_processing_metrics() if self._push_text_frames: - # We send the original text after the audio. This way, if we are - # interrupted, the text is not added to the assistant context. - frame = TTSTextFrame(text, aggregated_by=src_frame.aggregated_by) + # In TTS services that support word timestamps, the TTSTextFrames + # are pushed as words are spoken. However, in the case where the TTS service + # does not support word timestamps (i.e. _push_text_frames is True), we send + # the original (non-transformed) text after the TTS generation has completed. + # This way, if we are interrupted, the text is not added to the assistant + # context and the context that IS added does not include TTS-specific tags + # or transformations. + frame = TTSTextFrame(text, aggregated_by=type) frame.includes_inter_frame_spaces = includes_inter_frame_spaces await self.push_frame(frame) diff --git a/tests/test_piper_tts.py b/tests/test_piper_tts.py index 75893f93f..a006f555c 100644 --- a/tests/test_piper_tts.py +++ b/tests/test_piper_tts.py @@ -13,6 +13,7 @@ import pytest from aiohttp import web from pipecat.frames.frames import ( + AggregatedTextFrame, ErrorFrame, TTSAudioRawFrame, TTSSpeakFrame, @@ -74,6 +75,7 @@ async def test_run_piper_tts_success(aiohttp_client): ] expected_returned_frames = [ + AggregatedTextFrame, TTSStartedFrame, TTSAudioRawFrame, TTSAudioRawFrame, @@ -121,7 +123,7 @@ async def test_run_piper_tts_error(aiohttp_client): TTSSpeakFrame(text="Error case."), ] - expected_down_frames = [TTSStoppedFrame, TTSTextFrame] + expected_down_frames = [AggregatedTextFrame, TTSStoppedFrame, TTSTextFrame] expected_up_frames = [ErrorFrame] From e1528d0f0c737a43d68099d0f29cea5a9a9328f5 Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Mon, 17 Nov 2025 16:59:27 -0500 Subject: [PATCH 091/110] Added support to TTS services to skip sending text to the... the actual TTS service to be spoken based on its aggregation type. --- CHANGELOG.md | 4 ++++ src/pipecat/services/tts_service.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95cc4d564..89f6d38e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -159,6 +159,10 @@ Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and encompassing text should be added to the LLM context (by the LLM assistant aggregator). It defaults to `True`. + - `TTSService` base class updates: + - `TTSService`s now accept a new `skip_aggregator_types` to avoid speaking certain aggregation + types (now determined/returned by the aggregator) + ### Deprecated - The `api_key` parameter in `GeminiTTSService` is deprecated. Use diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 2267ee29f..637077f10 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -103,6 +103,8 @@ class TTSService(AIService): sample_rate: Optional[int] = None, # Text aggregator to aggregate incoming tokens and decide when to push to the TTS. text_aggregator: Optional[BaseTextAggregator] = None, + # Types of text aggregations that should not be spoken. + skip_aggregator_types: Optional[List[str]] = [], # Text filter executed after text has been aggregated. text_filters: Optional[Sequence[BaseTextFilter]] = None, text_filter: Optional[BaseTextFilter] = None, @@ -121,6 +123,7 @@ class TTSService(AIService): silence_time_s: Duration of silence to push when push_silence_after_stop is True. pause_frame_processing: Whether to pause frame processing during audio generation. sample_rate: Output sample rate for generated audio. + skip_aggregator_types: List of aggregation types that should not be spoken. text_aggregator: Custom text aggregator for processing incoming text. .. deprecated:: 0.0.95 @@ -158,6 +161,7 @@ class TTSService(AIService): DeprecationWarning, ) + self._skip_aggregator_types: List[str] = skip_aggregator_types or [] self._text_filters: Sequence[BaseTextFilter] = text_filters or [] self._transport_destination: Optional[str] = transport_destination self._tracing_enabled: bool = False @@ -501,6 +505,12 @@ class TTSService(AIService): type = src_frame.aggregated_by text = src_frame.text + # Skip sending to TTS if the aggregation type is in the skip list. Simply + # push the original frame downstream. + if type in self._skip_aggregator_types: + await self.push_frame(src_frame) + return + # Remove leading newlines only text = text.lstrip("\n") From ecbc41045c3241bdfb043bdf2f51691cba3e2088 Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Mon, 17 Nov 2025 18:14:45 -0500 Subject: [PATCH 092/110] Added ability to transform text just-in-time before it gets sent to the TTS --- CHANGELOG.md | 6 +++ src/pipecat/services/tts_service.py | 65 ++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89f6d38e3..07d922632 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -162,6 +162,12 @@ Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and - `TTSService` base class updates: - `TTSService`s now accept a new `skip_aggregator_types` to avoid speaking certain aggregation types (now determined/returned by the aggregator) + - Introduced the ability to do a just-in-time transform of text before it gets sent to the + TTS service via callbacks you can set up via a new init field, `text_transforms` or a new + method `add_text_transformer()`. This makes it possible to do things like introduce + TTS-specific tags for spelling or emotion or change the pronunciation of something on the + fly. `remove_text_transformer` has also been added to support removing a registered + transform callback. ### Deprecated diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 637077f10..8d7c4e6bb 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -12,6 +12,8 @@ from typing import ( Any, AsyncGenerator, AsyncIterator, + Awaitable, + Callable, Dict, List, Mapping, @@ -105,6 +107,14 @@ class TTSService(AIService): text_aggregator: Optional[BaseTextAggregator] = None, # Types of text aggregations that should not be spoken. skip_aggregator_types: Optional[List[str]] = [], + # A list of callables to transform text before just before sending it to TTS. + # Each callable takes the aggregated text and its type, and returns the transformed text. + # To register, provide a list of tuples of (aggregation_type | '*', transform_function). + text_transforms: Optional[ + List[ + Tuple[AggregationType | str, Callable[[str, str | AggregationType], Awaitable[str]]] + ] + ] = None, # Text filter executed after text has been aggregated. text_filters: Optional[Sequence[BaseTextFilter]] = None, text_filter: Optional[BaseTextFilter] = None, @@ -123,12 +133,17 @@ class TTSService(AIService): silence_time_s: Duration of silence to push when push_silence_after_stop is True. pause_frame_processing: Whether to pause frame processing during audio generation. sample_rate: Output sample rate for generated audio. - skip_aggregator_types: List of aggregation types that should not be spoken. text_aggregator: Custom text aggregator for processing incoming text. .. deprecated:: 0.0.95 Use an LLMTextProcessor before the TTSService for custom text aggregation. + skip_aggregator_types: List of aggregation types that should not be spoken. + text_transforms: A list of callables to transform text before just before sending it + to TTS. Each callable takes the aggregated text and its type, and returns the + transformed text. To register, provide a list of tuples of + (aggregation_type | '*', transform_function). + text_filters: Sequence of text filters to apply after aggregation. text_filter: Single text filter (deprecated, use text_filters). @@ -162,6 +177,10 @@ class TTSService(AIService): ) self._skip_aggregator_types: List[str] = skip_aggregator_types or [] + self._text_transforms: List[ + Tuple[AggregationType | str, Callable[[str, AggregationType | str], Awaitable[str]]] + ] = text_transforms or [] + # TODO: Deprecate _text_filters when added to LLMTextProcessor self._text_filters: Sequence[BaseTextFilter] = text_filters or [] self._transport_destination: Optional[str] = transport_destination self._tracing_enabled: bool = False @@ -301,6 +320,39 @@ class TTSService(AIService): await self.cancel_task(self._stop_frame_task) self._stop_frame_task = None + def add_text_transformer( + self, + transform_function: Callable[[str, AggregationType | str], Awaitable[str]], + aggregation_type: AggregationType | str = "*", + ): + """Transform text for a specific aggregation type. + + Args: + transform_function: The function to apply for transformation. This function should take + the text and aggregation type as input and return the transformed text. + Ex.: async def my_transform(text: str, aggregation_type: str) -> str: + aggregation_type: The type of aggregation to transform. This value defaults to "*" indicating + the function should handle all text before sending to TTS. + """ + self._text_transforms.append((aggregation_type, transform_function)) + + def remove_text_transformer( + self, + transform_function: Callable[[str, AggregationType | str], Awaitable[str]], + aggregation_type: AggregationType | str = "*", + ): + """Remove a text transformer for a specific aggregation type. + + Args: + transform_function: The function to remove. + aggregation_type: The type of aggregation to remove the transformer for. + """ + self._text_transforms = [ + (agg_type, func) + for agg_type, func in self._text_transforms + if not (agg_type == aggregation_type and func == transform_function) + ] + async def _update_settings(self, settings: Mapping[str, Any]): for key, value in settings.items(): if key in self._settings: @@ -542,7 +594,16 @@ class TTSService(AIService): src_frame.append_to_context = False await self.push_frame(src_frame) - await self.process_generator(self.run_tts(text)) + # Note: Text transformations are meant to only affect the text sent to the TTS for + # TTS-specific purposes. This allows for explicit TTS modifications (e.g., inserting + # TTS supported tags for spelling or emotion or replacing an @ with "at"). For TTS + # services that support word-level timestamps, this CAN affect the resulting context + # since the TTSTextFrames are generated from the TTS output stream + transformed_text = text + for aggregation_type, transform in self._text_transforms: + if aggregation_type == type or aggregation_type == "*": + transformed_text = await transform(transformed_text, type) + await self.process_generator(self.run_tts(transformed_text)) await self.stop_processing_metrics() From 4f30a48ecd292913aa1b4c56367af0828abe6851 Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Mon, 17 Nov 2025 16:52:48 -0500 Subject: [PATCH 093/110] Rime and Cartesia TTS Updates: `CartesiaTTSService`: - Modified use of custom default text_aggregator to avoid deprecation warnings and push users towards use of transformers or the `LLMTextProcessor` - Added convenience methods for taking advantage of Cartesia's SSML tags: spell, emotion, pauses, volume, and speed. `RimeTTSService`: - Modified use of custom default text_aggregator to avoid deprecation warnings and push users towards use of transformers or the `LLMTextProcessor` - Added convenience methods for taking advantage of Rime's customization options: spell, pauses, pronunciations, and inline speed control. --- CHANGELOG.md | 12 ++++ src/pipecat/services/cartesia/tts.py | 104 ++++++++++++++++++++++++++- src/pipecat/services/rime/tts.py | 45 +++++++++++- 3 files changed, 157 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07d922632..8a54c5d1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -169,6 +169,18 @@ Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and fly. `remove_text_transformer` has also been added to support removing a registered transform callback. + - Updated `CartesiaTTSService`: + - Modified use of custom default text_aggregator to avoid deprecation warnings and push users + towards use of transformers or the `LLMTextProcessor` + - Added convenience methods for taking advantage of Cartesia's SSML tags: spell, emotion, + pauses, volume, and speed. + + - Updated `RimeTTSService`: + - Modified use of custom default text_aggregator to avoid deprecation warnings and push users + towards use of transformers or the `LLMTextProcessor` + - Added convenience methods for taking advantage of Rime's customization options: spell, + pauses, pronunciations, and inline speed control. + ### Deprecated - The `api_key` parameter in `GeminiTTSService` is deprecated. Use diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index f8881200c..d42802cf2 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -10,7 +10,8 @@ import base64 import json import uuid import warnings -from typing import AsyncGenerator, List, Literal, Optional, Union +from enum import Enum +from typing import AsyncGenerator, List, Literal, Optional from loguru import logger from pydantic import BaseModel, Field @@ -125,6 +126,72 @@ def language_to_cartesia_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +class CartesiaEmotion(str, Enum): + """Predefined Emotions supported by Cartesia.""" + + # Primary emotions supported by Cartesia + NEUTRAL = "neutral" + ANGRY = "angry" + EXCITED = "excited" + CONTENT = "content" + SAD = "sad" + SCARED = "scared" + # Additional emotions supported by Cartesia + HAPPY = "happy" + ENTHUSIASTIC = "enthusiastic" + ELATED = "elated" + EUPHORIC = "euphoric" + TRIUMPHANT = "triumphant" + AMAZED = "amazed" + SURPRISED = "surprised" + FLIRTATIOUS = "flirtatious" + JOKING_COMEDIC = "joking/comedic" + CURIOUS = "curious" + PEACEFUL = "peaceful" + SERENE = "serene" + CALM = "calm" + GRATEFUL = "grateful" + AFFECTIONATE = "affectionate" + TRUST = "trust" + SYMPATHETIC = "sympathetic" + ANTICIPATION = "anticipation" + MYSTERIOUS = "mysterious" + MAD = "mad" + OUTRAGED = "outraged" + FRUSTRATED = "frustrated" + AGITATED = "agitated" + THREATENED = "threatened" + DISGUSTED = "disgusted" + CONTEMPT = "contempt" + ENVIOUS = "envious" + SARCASTIC = "sarcastic" + IRONIC = "ironic" + DEJECTED = "dejected" + MELANCHOLIC = "melancholic" + DISAPPOINTED = "disappointed" + HURT = "hurt" + GUILTY = "guilty" + BORED = "bored" + TIRED = "tired" + REJECTED = "rejected" + NOSTALGIC = "nostalgic" + WISTFUL = "wistful" + APOLOGETIC = "apologetic" + HESITANT = "hesitant" + INSECURE = "insecure" + CONFUSED = "confused" + RESIGNED = "resigned" + ANXIOUS = "anxious" + PANICKED = "panicked" + ALARMED = "alarmed" + PROUD = "proud" + CONFIDENT = "confident" + DISTANT = "distant" + SKEPTICAL = "skeptical" + CONTEMPLATIVE = "contemplative" + DETERMINED = "determined" + + class CartesiaTTSService(AudioContextWordTTSService): """Cartesia TTS service with WebSocket streaming and word timestamps. @@ -182,6 +249,10 @@ class CartesiaTTSService(AudioContextWordTTSService): container: Audio container format. params: Additional input parameters for voice customization. text_aggregator: Custom text aggregator for processing input text. + + .. deprecated:: 0.0.95 + Use an LLMTextProcessor before the TTSService for custom text aggregation. + aggregate_sentences: Whether to aggregate sentences within the TTSService. **kwargs: Additional arguments passed to the parent service. """ @@ -200,10 +271,18 @@ class CartesiaTTSService(AudioContextWordTTSService): push_text_frames=False, pause_frame_processing=True, sample_rate=sample_rate, - text_aggregator=text_aggregator or SkipTagsAggregator([("", "")]), + text_aggregator=text_aggregator, **kwargs, ) + if not text_aggregator: + # Always skip tags added for spelled-out text + # Note: This is primarily to support backwards compatibility. + # The preferred way of taking advantage of Cartesia SSML Tags is + # to use an LLMTextProcessor and/or a text_transformer to identify + # and insert these tags for the purpose of the TTS service alone. + self._text_aggregator = SkipTagsAggregator([("", "")]) + params = params or CartesiaTTSService.InputParams() self._api_key = api_key @@ -257,6 +336,27 @@ class CartesiaTTSService(AudioContextWordTTSService): """ return language_to_cartesia_language(language) + # A set of Cartesia-specific helpers for text transformations + def SPELL(text: str) -> str: + """Wrap text in Cartesia spell tag.""" + return f"{text}" + + def EMOTION_TAG(emotion: CartesiaEmotion) -> str: + """Convenience method to create an emotion tag.""" + return f'' + + def PAUSE_TAG(seconds: float) -> str: + """Convenience method to create a pause tag.""" + return f'' + + def VOLUME_TAG(volume: float) -> str: + """Convenience method to create a volume tag.""" + return f'' + + def SPEED_TAG(speed: float) -> str: + """Convenience method to create a speed tag.""" + return f'' + def _is_cjk_language(self, language: str) -> bool: """Check if the given language is CJK (Chinese, Japanese, Korean). diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 7b62f20fa..c9f461350 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -113,6 +113,10 @@ class RimeTTSService(AudioContextWordTTSService): sample_rate: Audio sample rate in Hz. params: Additional configuration parameters. text_aggregator: Custom text aggregator for processing input text. + + .. deprecated:: 0.0.95 + Use an LLMTextProcessor before the TTSService for custom text aggregation. + aggregate_sentences: Whether to aggregate sentences within the TTSService. **kwargs: Additional arguments passed to parent class. """ @@ -123,10 +127,17 @@ class RimeTTSService(AudioContextWordTTSService): push_stop_frames=True, pause_frame_processing=True, sample_rate=sample_rate, - text_aggregator=text_aggregator or SkipTagsAggregator([("spell(", ")")]), **kwargs, ) + if not text_aggregator: + # Always skip tags added for spelled-out text + # Note: This is primarily to support backwards compatibility. + # The preferred way of taking advantage of Rime spelling is + # to use an LLMTextProcessor and/or a text_transformer to identify + # and insert these tags for the purpose of the TTS service alone. + self._text_aggregator = SkipTagsAggregator([("spell(", ")")]) + params = params or RimeTTSService.InputParams() # Store service configuration @@ -152,6 +163,7 @@ class RimeTTSService(AudioContextWordTTSService): self._context_id = None # Tracks current turn self._receive_task = None self._cumulative_time = 0 # Accumulates time across messages + self._extra_msg_fields = {} # Extra fields for next message def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -181,6 +193,31 @@ class RimeTTSService(AudioContextWordTTSService): self._model = model await super().set_model(model) + # A set of Rime-specific helpers for text transformations + def SPELL(text: str) -> str: + """Wrap text in Rime spell function.""" + return f"spell({text})" + + def PAUSE_TAG(seconds: float) -> str: + """Convenience method to create a pause tag.""" + return f"<{seconds * 1000}>" + + def PRONOUNCE(self, text: str, word: str, phoneme: str) -> str: + """Convenience method to support Rime's custom pronunciations feature. + + https://docs.rime.ai/api-reference/custom-pronunciation + """ + self._extra_msg_fields["phonemizeBetweenBrackets"] = True + return text.replace(word, f"{phoneme}") + + def INLINE_SPEED(self, text: str, speed: float) -> str: + """Convenience method to support inline speeds.""" + if not self._extra_msg_fields: + self._extra_msg_fields = {} + speed_vals = self._extra_msg_fields.get("inlineSpeedAlpha", "").split(",") + self._extra_msg_fields["inlineSpeedAlpha"] = ",".join(speed_vals + [str(speed)]) + return f"[{text}]" + async def _update_settings(self, settings: Mapping[str, Any]): """Update service settings and reconnect if voice changed.""" prev_voice = self._voice_id @@ -193,7 +230,11 @@ class RimeTTSService(AudioContextWordTTSService): def _build_msg(self, text: str = "") -> dict: """Build JSON message for Rime API.""" - return {"text": text, "contextId": self._context_id} + msg = {"text": text, "contextId": self._context_id} + if self._extra_msg_fields: + msg |= self._extra_msg_fields + self._extra_msg_fields = {} + return msg def _build_clear_msg(self) -> dict: """Build clear operation message.""" From 8b8b57b09ca5598c7e5b62bb1ac570ef87124738 Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Mon, 17 Nov 2025 21:44:23 -0500 Subject: [PATCH 094/110] Introduced new bot-output RTVI event to provide... a best effort version of the bot's output - The `RTVIObserver` now emits `bot-output` messages based off the new `AggregatedTextFrame`s (`bot-tts-text` and `bot-llm-text` are still supported and generated, but `bot-transcript` is now deprecated in lieu of this new, more thorough, message). - The new `RTVIBotOutputMessage` includes the fields: - `spoken`: A boolean indicating whether the text was spoken by TTS - `aggregated_by`: A string representing how the text was aggregated ("sentence", "word", "my custom aggregation") - Introduced new fields to `RTVIObserver` to support the new `bot-output` messaging: - `bot_output_enabled`: Defaults to True. Set to false to disable bot-output messages. - `skip_aggregator_types`: Defaults to `None`. Set to a list of strings that match aggregation types that should not be included in bot-output messages. (Ex. `credit_card`) --- CHANGELOG.md | 17 +++ src/pipecat/processors/frameworks/rtvi.py | 150 ++++++++++++++++++---- 2 files changed, 145 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a54c5d1d..4f213e1db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 TTS's internal text_aggregator, but instead, insert this processor between your LLM and TTS in the pipeline. +- New `bot-output` RTVI message to represent what the bot actually "says". + - The `RTVIObserver` now emits `bot-output` messages based off the new `AggregatedTextFrame`s + (`bot-tts-text` and `bot-llm-text` are still supported and generated, but `bot-transcript` is + now deprecated in lieu of this new, more thorough, message). + - The new `RTVIBotOutputMessage` includes the fields: + - `spoken`: A boolean indicating whether the text was spoken by TTS + - `aggregated_by`: A string representing how the text was aggregated ("sentence", "word", + "my custom aggregation") + - Introduced new fields to `RTVIObserver` to support the new `bot-output` messaging: + - `bot_output_enabled`: Defaults to True. Set to false to disable bot-output messages. + - `skip_aggregator_types`: Defaults to `None`. Set to a list of strings that match + aggregation types that should not be included in bot-output messages. (Ex. `credit_card`) + ### Changed - ⚠️ Breaking change: `LLMContext.create_image_message()`, @@ -194,6 +207,10 @@ use `test_normalization` instead. behavior, but if you want to override the aggregation behavior, you should use the new processor. +- The RTVI `bot-transcription` event is deprecated in favor of the new `bot-output` + message which is the canonical representation of bot output (spoken or not). The code + still emits a transcription message for backwards compatibility while transition occurs. + ### Fixed - Fixed a `SimliVideoService` connection issue. diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index f04cbd395..7bd84fb42 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -24,6 +24,7 @@ from typing import ( Literal, Mapping, Optional, + Tuple, Union, ) @@ -32,6 +33,8 @@ from pydantic import BaseModel, Field, PrivateAttr, ValidationError from pipecat.audio.utils import calculate_audio_volume from pipecat.frames.frames import ( + AggregatedTextFrame, + AggregationType, BotStartedSpeakingFrame, BotStoppedSpeakingFrame, CancelFrame, @@ -704,6 +707,29 @@ class RTVITextMessageData(BaseModel): text: str +class RTVIBotOutputMessageData(RTVITextMessageData): + """Data for bot output RTVI messages. + + Extends RTVITextMessageData to include metadata about the output. + """ + + spoken: bool = False # Indicates if the text has been spoken by TTS + aggregated_by: AggregationType | str + # Indicates what form the text is in (e.g., by word, sentence, etc.) + + +class RTVIBotOutputMessage(BaseModel): + """Message containing bot output text. + + An event meant to holistically represent what the bot is outputting, + along with metadata about the output and if it has been spoken. + """ + + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL + type: Literal["bot-output"] = "bot-output" + data: RTVIBotOutputMessageData + + class RTVIBotTranscriptionMessage(BaseModel): """Message containing bot transcription text. @@ -896,6 +922,7 @@ class RTVIObserverParams: Parameter `errors_enabled` is deprecated. Error messages are always enabled. Parameters: + bot_output_enabled: Indicates if bot output messages should be sent. bot_llm_enabled: Indicates if the bot's LLM messages should be sent. bot_tts_enabled: Indicates if the bot's TTS messages should be sent. bot_speaking_enabled: Indicates if the bot's started/stopped speaking messages should be sent. @@ -907,9 +934,17 @@ class RTVIObserverParams: metrics_enabled: Indicates if metrics messages should be sent. system_logs_enabled: Indicates if system logs should be sent. errors_enabled: [Deprecated] Indicates if errors messages should be sent. + skip_aggregator_types: List of aggregation types to skip sending as tts/output messages. + Note: if using this to avoid sending secure information, be sure to also disable + bot_llm_enabled to avoid leaking through LLM messages. + bot_output_transforms: A list of callables to transform text before just before sending it + to TTS. Each callable takes the aggregated text and its type, and returns the + transformed text. To register, provide a list of tuples of + (aggregation_type | '*', transform_function). audio_level_period_secs: How often audio levels should be sent if enabled. """ + bot_output_enabled: bool = True bot_llm_enabled: bool = True bot_tts_enabled: bool = True bot_speaking_enabled: bool = True @@ -921,6 +956,15 @@ class RTVIObserverParams: metrics_enabled: bool = True system_logs_enabled: bool = False errors_enabled: Optional[bool] = None + skip_aggregator_types: Optional[List[AggregationType | str]] = None + bot_output_transforms: Optional[ + List[ + Tuple[ + AggregationType | str, + Callable[[str, AggregationType | str], Awaitable[str]], + ] + ] + ] = None audio_level_period_secs: float = 0.15 @@ -973,8 +1017,45 @@ class RTVIObserver(BaseObserver): DeprecationWarning, ) + self._aggregation_transforms: List[ + Tuple[AggregationType | str, Callable[[str, AggregationType | str], Awaitable[str]]] + ] = self._params.bot_output_transforms or [] + + def add_bot_output_transformer( + self, + transform_function: Callable[[str, AggregationType | str], Awaitable[str]], + aggregation_type: AggregationType | str = "*", + ): + """Transform text for a specific aggregation type before sending as Bot Output or TTS. + + Args: + transform_function: The function to apply for transformation. This function should take + the text and aggregation type as input and return the transformed text. + Ex.: async def my_transform(text: str, aggregation_type: str) -> str: + aggregation_type: The type of aggregation to transform. This value defaults to "*" to + handle all text before sending to the client. + """ + self._aggregation_transforms.append((aggregation_type, transform_function)) + + def remove_bot_output_transformer( + self, + transform_function: Callable[[str, AggregationType | str], Awaitable[str]], + aggregation_type: AggregationType | str = "*", + ): + """Remove a text transformer for a specific aggregation type. + + Args: + transform_function: The function to remove. + aggregation_type: The type of aggregation to remove the transformer for. + """ + self._aggregation_transforms = [ + (agg_type, func) + for agg_type, func in self._aggregation_transforms + if not (agg_type == aggregation_type and func == transform_function) + ] + async def _logger_sink(self, message): - """Logger sink so we cna send system logs to RTVI clients.""" + """Logger sink so we can send system logs to RTVI clients.""" message = RTVISystemLogMessage(data=RTVITextMessageData(text=message)) await self.send_rtvi_message(message) @@ -1048,12 +1129,15 @@ class RTVIObserver(BaseObserver): await self.send_rtvi_message(RTVIBotTTSStartedMessage()) elif isinstance(frame, TTSStoppedFrame) and self._params.bot_tts_enabled: await self.send_rtvi_message(RTVIBotTTSStoppedMessage()) - elif isinstance(frame, TTSTextFrame) and self._params.bot_tts_enabled: - if isinstance(src, BaseOutputTransport): - message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text)) - await self.send_rtvi_message(message) - else: + elif isinstance(frame, AggregatedTextFrame) and ( + self._params.bot_output_enabled or self._params.bot_tts_enabled + ): + if isinstance(frame, TTSTextFrame) and not isinstance(src, BaseOutputTransport): + # This check is to make sure we handle the frame when it has gone + # through the transport and has correct timing. mark_as_seen = False + else: + await self._handle_aggregated_llm_text(frame) elif isinstance(frame, MetricsFrame) and self._params.metrics_enabled: await self._handle_metrics(frame) elif isinstance(frame, RTVIServerMessageFrame): @@ -1084,15 +1168,6 @@ class RTVIObserver(BaseObserver): if mark_as_seen: self._frames_seen.add(frame.id) - async def _push_bot_transcription(self): - """Push accumulated bot transcription as a message.""" - if len(self._bot_transcription) > 0: - message = RTVIBotTranscriptionMessage( - data=RTVITextMessageData(text=self._bot_transcription) - ) - await self.send_rtvi_message(message) - self._bot_transcription = "" - async def _handle_interruptions(self, frame: Frame): """Handle user speaking interruption frames.""" message = None @@ -1115,14 +1190,45 @@ class RTVIObserver(BaseObserver): if message: await self.send_rtvi_message(message) + async def _handle_aggregated_llm_text(self, frame: AggregatedTextFrame): + """Handle aggregated LLM text output frames.""" + # Skip certain aggregator types if configured to do so. + if ( + self._params.skip_aggregator_types + and frame.aggregated_by in self._params.skip_aggregator_types + ): + return + + text = frame.text + type = frame.aggregated_by + for aggregation_type, transform in self._aggregation_transforms: + if aggregation_type == type or aggregation_type == "*": + text = await transform(text, type) + + isTTS = isinstance(frame, TTSTextFrame) + if self._params.bot_output_enabled: + message = RTVIBotOutputMessage( + data=RTVIBotOutputMessageData(text=text, spoken=isTTS, aggregated_by=type) + ) + await self.send_rtvi_message(message) + + if isTTS and self._params.bot_tts_enabled: + tts_message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=text)) + await self.send_rtvi_message(tts_message) + async def _handle_llm_text_frame(self, frame: LLMTextFrame): """Handle LLM text output frames.""" message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text)) await self.send_rtvi_message(message) + # TODO (mrkb): Remove all this logic when we fully deprecate bot-transcription messages. self._bot_transcription += frame.text - if match_endofsentence(self._bot_transcription): - await self._push_bot_transcription() + + if match_endofsentence(self._bot_transcription) and len(self._bot_transcription) > 0: + await self.send_rtvi_message( + RTVIBotTranscriptionMessage(data=RTVITextMessageData(text=self._bot_transcription)) + ) + self._bot_transcription = "" async def _handle_user_transcriptions(self, frame: Frame): """Handle user transcription frames.""" @@ -1248,7 +1354,7 @@ class RTVIProcessor(FrameProcessor): # Default to 0.3.0 which is the last version before actually having a # "client-version". self._client_version = [0, 3, 0] - self._skip_tts: bool = False # Keep in sync with llm_service.py + self._llm_skip_tts: bool = False # Keep in sync with llm_service.py's configuration. self._registered_actions: Dict[str, RTVIAction] = {} self._registered_services: Dict[str, RTVIService] = {} @@ -1441,7 +1547,7 @@ class RTVIProcessor(FrameProcessor): elif isinstance(frame, RTVIActionFrame): await self._action_queue.put(frame) elif isinstance(frame, LLMConfigureOutputFrame): - self._skip_tts = frame.skip_tts + self._llm_skip_tts = frame.skip_tts await self.push_frame(frame, direction) # Other frames else: @@ -1697,9 +1803,9 @@ class RTVIProcessor(FrameProcessor): opts = data.options if data.options is not None else RTVISendTextOptions() if opts.run_immediately: await self.interrupt_bot() - cur_skip_tts = self._skip_tts + cur_llm_skip_tts = self._llm_skip_tts should_skip_tts = not opts.audio_response - toggle_skip_tts = cur_skip_tts != should_skip_tts + toggle_skip_tts = cur_llm_skip_tts != should_skip_tts if toggle_skip_tts: output_frame = LLMConfigureOutputFrame(skip_tts=should_skip_tts) await self.push_frame(output_frame) @@ -1709,7 +1815,7 @@ class RTVIProcessor(FrameProcessor): ) await self.push_frame(text_frame) if toggle_skip_tts: - output_frame = LLMConfigureOutputFrame(skip_tts=cur_skip_tts) + output_frame = LLMConfigureOutputFrame(skip_tts=cur_llm_skip_tts) await self.push_frame(output_frame) async def _handle_update_context(self, data: RTVIAppendToContextData): From 3c3141796a8ece855817e6db77c7c73aaf77a392 Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Tue, 18 Nov 2025 15:01:26 -0500 Subject: [PATCH 095/110] Overlooked Changelog updates --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f213e1db..7ee108c11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -181,6 +181,11 @@ Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and TTS-specific tags for spelling or emotion or change the pronunciation of something on the fly. `remove_text_transformer` has also been added to support removing a registered transform callback. + - TTS services push `AggregatedTextFrame` in addition to `TTSTextFrame`s when either an + aggregation occurs that should not be spoken or when the TTS service supports word-by-word + timestamping. In the latter case, the `TTSService` preliminarily generates an + `AggregatedTextFrame`, aggregated by sentence to generate the full sentence content as early + as possible. - Updated `CartesiaTTSService`: - Modified use of custom default text_aggregator to avoid deprecation warnings and push users @@ -211,6 +216,10 @@ use `test_normalization` instead. message which is the canonical representation of bot output (spoken or not). The code still emits a transcription message for backwards compatibility while transition occurs. +- Deprecated `add_pattern_pair` in the `PatternPairAggregator` which takes a `pattern_id` + and `remove_match` field in favor of the new `add_pattern` method which takes a `type` and an + `action` + ### Fixed - Fixed a `SimliVideoService` connection issue. From a6202c4d1a7c7a7f12f6e382d267efeef2c67853 Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Thu, 20 Nov 2025 14:58:53 -0500 Subject: [PATCH 096/110] Fixed CHANGELOG post rebase --- CHANGELOG.md | 254 +++++++++--------- .../utils/text/base_text_aggregator.py | 2 +- 2 files changed, 132 insertions(+), 124 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ee108c11..c9208b8d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,34 +11,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added optional speaking rate control to `InworldTTSService`. -### Changed - -- Updated `daily-python` to 0.22.0. - -### Fixed - -- Fixed `InworldTTSService` audio config payload to use camelCase keys expected - by the Inworld API. - -## [0.0.95] - 2025-11-18 - -### Added - -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no - ONNX dependency or added processing complexity. - -- Added a watchdog to `DeepgramFluxSTTService` to prevent dangling tasks in case the - user was speaking and we stop receiving audio. - -- Introduced a minimum confidence parameter in `DeepgramFluxSTTService` to avoid - generating transcriptions below a defined threshold. - -- Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT - service from ElevenLabs. - -- Added word-level timestamps support to Hume TTS service - - Introduced a new `AggregatedTextFrame` type to support passing text along with an `aggregated_by` field to describe the type of text included. `TTSTextFrame`s now inherit from `AggregatedTextFrame`. With this inheritance, an observer can watch for @@ -71,6 +43,137 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `bot_output_enabled`: Defaults to True. Set to false to disable bot-output messages. - `skip_aggregator_types`: Defaults to `None`. Set to a list of strings that match aggregation types that should not be included in bot-output messages. (Ex. `credit_card`) + - Introduced new methods, `add_text_transformer()` and `remove_text_transformer()`, to + `RTVIObserver` to support providing (and subsequently removing) callbacks for various types of + aggregations (or all aggregations with `*`) that can modify the text before being sent as a + `bot-output` or `tts-text` message. (Think obscuring the credit card or inserting extra detail + the client might want that the context doesn't need.) + +### Changed + +- Updated `daily-python` to 0.22.0. + +- `BaseTextAggregator` changes: + Modified the BaseTextAggregator type so that when text gets aggregated, metadata can + be associated with it. Currently, that just means a `type`, so that the aggregation + can be classified or described. Changes made to support this: + - ⚠️ IMPORTANT: Aggregators are now expected to strip leading/trailing white space + characters before returning their aggregation from `aggregation()` or `.text`. This + way all aggregators have a consistent contract allowing downstream use to know how + to stitch aggregations back together. + - Introduced a new `Aggregation` dataclass to represent both the aggregated `text` and + a string identifying the `type` of aggregation (ex. "sentence", "word", "my custom + aggregation") + - ⚠️ Breaking change: `BaseTextAggregator.text` now returns an `Aggregation` (instead of `str`). + To update: `aggregated_text = myAggregator.text` -> `aggregated_text = myAggregator.text.text` + - ⚠️ Breaking change: `BaseTextAggregator.aggregate()` now returns `Optional[Aggregation]` + (instead of `Optional[str]`). To update: + ``` + aggregation = myAggregator.aggregate(text) + if (aggregation): + print(f"successfully aggregated text: {aggregation.text}") // instead of {aggregation} + ``` + - `SimpleTextAggregator`, `SkipTagsAggregator`, `PatternPairAggregator` updated to + produce/consume `Aggregation` objects. + - All uses of the above Aggregators have been updated accordingly. + +- Augmented the `PatternPairAggregator` so that matched patterns can be treated as their own + aggregation, taking advantage of the new. To that end: + - Introduced a new, preferred version of `add_pattern` to support a new option for treating a + match as a separate aggregation returned from `aggregate()`. This replaces the now + deprecated `add_pattern_pair` method and you provide a `MatchAction` in lieu of the `remove_match` field. + - `MatchAction` enum: `REMOVE`, `KEEP`, `AGGREGATE`, allowing customization for how + a match should be handled. + - `REMOVE`: The text along with its delimiters will be removed from the streaming text. + Sentence aggregation will continue on as if this text did not exist. + - `KEEP`: The delimiters will be removed, but the content between them will be kept. + Sentence aggregation will continue on with the internal text included. + - `AGGREGATE`: The delimiters will be removed and the content between will be treated + as a separate aggregation. Any text before the start of the pattern will be + returned early, whether or not a complete sentence was found. Then the pattern + will be returned. Then the aggregation will continue on sentence matching after + the closing delimiter is found. The content between the delimiters is not + aggregated by sentence. It is aggregated as one single block of text. + - `PatternMatch` now extends `Aggregation` and provides richer info to handlers. + - ⚠️ Breaking change: The `PatternMatch` type returned to handlers registered via `on_pattern_match` + has been updated to subclass from the new `Aggregation` type, which means that `content` + has been replaced with `text` and `pattern_id` has been replaced with `type`: + ``` + async dev on_match_tag(match: PatternMatch): + pattern = match.type # instead of match.pattern_id + text = match.text # instead of match.content + ``` + +- `TextFrame` now includes the field `append_to_context` to support setting whether or not the + encompassing text should be added to the LLM context (by the LLM assistant aggregator). It + defaults to `True`. + +- `TTSService` base class updates: + - `TTSService`s now accept a new `skip_aggregator_types` to avoid speaking certain aggregation + types (now determined/returned by the aggregator) + - Introduced the ability to do a just-in-time transform of text before it gets sent to the + TTS service via callbacks you can set up via a new init field, `text_transforms` or a new + method `add_text_transformer()`. This makes it possible to do things like introduce + TTS-specific tags for spelling or emotion or change the pronunciation of something on the + fly. `remove_text_transformer` has also been added to support removing a registered + transform callback. + - TTS services push `AggregatedTextFrame` in addition to `TTSTextFrame`s when either an + aggregation occurs that should not be spoken or when the TTS service supports word-by-word + timestamping. In the latter case, the `TTSService` preliminarily generates an + `AggregatedTextFrame`, aggregated by sentence to generate the full sentence content as early + as possible. + +- Updated `CartesiaTTSService`: + - Modified use of custom default text_aggregator to avoid deprecation warnings and push users + towards use of transformers or the `LLMTextProcessor` + - Added convenience methods for taking advantage of Cartesia's SSML tags: spell, emotion, + pauses, volume, and speed. + +- Updated `RimeTTSService`: + - Modified use of custom default text_aggregator to avoid deprecation warnings and push users + towards use of transformers or the `LLMTextProcessor` + - Added convenience methods for taking advantage of Rime's customization options: spell, + pauses, pronunciations, and inline speed control. + +### Deprecated + +- The TTS constructor field, `text_aggregator` is deprecated in favor of the new + `LLMTextProcessor`. TTSServices still have an internal aggregator for support of default + behavior, but if you want to override the aggregation behavior, you should use the new + processor. + +- The RTVI `bot-transcription` event is deprecated in favor of the new `bot-output` + message which is the canonical representation of bot output (spoken or not). The code + still emits a transcription message for backwards compatibility while transition occurs. + +- Deprecated `add_pattern_pair` in the `PatternPairAggregator` which takes a `pattern_id` + and `remove_match` field in favor of the new `add_pattern` method which takes a `type` and an + `action` + + +### Fixed + +- Fixed `InworldTTSService` audio config payload to use camelCase keys expected + by the Inworld API. + +## [0.0.95] - 2025-11-18 + +### Added + +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no + ONNX dependency or added processing complexity. + +- Added a watchdog to `DeepgramFluxSTTService` to prevent dangling tasks in case the + user was speaking and we stop receiving audio. + +- Introduced a minimum confidence parameter in `DeepgramFluxSTTService` to avoid + generating transcriptions below a defined threshold. + +- Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT + service from ElevenLabs. + +- Added word-level timestamps support to Hume TTS service ### Changed @@ -117,88 +220,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and Tamil -- Added new emotions: calm and fluent -- `BaseTextAggregator` changes: - Modified the BaseTextAggregator type so that when text gets aggregated, metadata can - be associated with it. Currently, that just means a `type`, so that the aggregation - can be classified or described. Changes made to support this: - - **IMPORTANT**: Aggregators are now expected to strip leading/trailing white space - characters before returning their aggregation from `aggregation()` or `.text`. This - way all aggregators have a consistent contract allowing downstream use to know how - to stitch aggregations back together. - - Introduced a new `Aggregation` dataclass to represent both the aggregated `text` and - a string identifying the `type` of aggregation (ex. "sentence", "word", "my custom - aggregation") - - **BREAKING**: `BaseTextAggregator.text` now returns an `Aggregation` (instead of `str`). - To update: `aggregated_text = myAggregator.text` -> `aggregated_text = myAggregator.text.text` - - **BREAKING**: `BaseTextAggregator.aggregate()` now returns `Optional[Aggregation]` - (instead of `Optional[str]`). To update: - ``` - aggregation = myAggregator.aggregate(text) - if (aggregation): - print(f"successfully aggregated text: {aggregation.text}") // instead of {aggregation} - ``` - - `SimpleTextAggregator`, `SkipTagsAggregator`, `PatternPairAggregator` updated to - produce/consume `Aggregation` objects. - - All uses of the above Aggregators have been updated accordingly. - -- Augmented the `PatternPairAggregator` so that matched patterns can be treated as their own - aggregation, taking advantage of the new. To that end: - - Introduced a new, preferred version of `add_pattern` to support a new option for treating a - match as a separate aggregation returned from `aggregate()`. This replaces the now - deprecated `add_pattern_pair` method and you provide a `MatchAction` in lieu of the `remove_match` field. - - `MatchAction` enum: `REMOVE`, `KEEP`, `AGGREGATE`, allowing customization for how - a match should be handled. - - `REMOVE`: The text along with its delimiters will be removed from the streaming text. - Sentence aggregation will continue on as if this text did not exist. - - `KEEP`: The delimiters will be removed, but the content between them will be kept. - Sentence aggregation will continue on with the internal text included. - - `AGGREGATE`: The delimiters will be removed and the content between will be treated - as a separate aggregation. Any text before the start of the pattern will be - returned early, whether or not a complete sentence was found. Then the pattern - will be returned. Then the aggregation will continue on sentence matching after - the closing delimiter is found. The content between the delimiters is not - aggregated by sentence. It is aggregated as one single block of text. - - `PatternMatch` now extends `Aggregation` and provides richer info to handlers. - - **BREAKING**: The `PatternMatch` type returned to handlers registered via `on_pattern_match` - has been updated to subclass from the new `Aggregation` type, which means that `content` - has been replaced with `text` and `pattern_id` has been replaced with `type`: - ``` - async dev on_match_tag(match: PatternMatch): - pattern = match.type # instead of match.pattern_id - text = match.text # instead of match.content - ``` - - - `TextFrame` now includes the field `append_to_context` to support setting whether or not the - encompassing text should be added to the LLM context (by the LLM assistant aggregator). It - defaults to `True`. - - - `TTSService` base class updates: - - `TTSService`s now accept a new `skip_aggregator_types` to avoid speaking certain aggregation - types (now determined/returned by the aggregator) - - Introduced the ability to do a just-in-time transform of text before it gets sent to the - TTS service via callbacks you can set up via a new init field, `text_transforms` or a new - method `add_text_transformer()`. This makes it possible to do things like introduce - TTS-specific tags for spelling or emotion or change the pronunciation of something on the - fly. `remove_text_transformer` has also been added to support removing a registered - transform callback. - - TTS services push `AggregatedTextFrame` in addition to `TTSTextFrame`s when either an - aggregation occurs that should not be spoken or when the TTS service supports word-by-word - timestamping. In the latter case, the `TTSService` preliminarily generates an - `AggregatedTextFrame`, aggregated by sentence to generate the full sentence content as early - as possible. - - - Updated `CartesiaTTSService`: - - Modified use of custom default text_aggregator to avoid deprecation warnings and push users - towards use of transformers or the `LLMTextProcessor` - - Added convenience methods for taking advantage of Cartesia's SSML tags: spell, emotion, - pauses, volume, and speed. - - - Updated `RimeTTSService`: - - Modified use of custom default text_aggregator to avoid deprecation warnings and push users - towards use of transformers or the `LLMTextProcessor` - - Added convenience methods for taking advantage of Rime's customization options: spell, - pauses, pronunciations, and inline speed control. - ### Deprecated - The `api_key` parameter in `GeminiTTSService` is deprecated. Use @@ -207,19 +228,6 @@ Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and - `english_normalization` input parameter for `MiniMaxHttpTTSService` is deprecated, use `test_normalization` instead. -- The TTS constructor field, `text_aggregator` is deprecated in favor of the new - `LLMTextProcessor`. TTSServices still have an internal aggregator for support of default - behavior, but if you want to override the aggregation behavior, you should use the new - processor. - -- The RTVI `bot-transcription` event is deprecated in favor of the new `bot-output` - message which is the canonical representation of bot output (spoken or not). The code - still emits a transcription message for backwards compatibility while transition occurs. - -- Deprecated `add_pattern_pair` in the `PatternPairAggregator` which takes a `pattern_id` - and `remove_match` field in favor of the new `add_pattern` method which takes a `type` and an - `action` - ### Fixed - Fixed a `SimliVideoService` connection issue. diff --git a/src/pipecat/utils/text/base_text_aggregator.py b/src/pipecat/utils/text/base_text_aggregator.py index 07cb6c097..5c39ce769 100644 --- a/src/pipecat/utils/text/base_text_aggregator.py +++ b/src/pipecat/utils/text/base_text_aggregator.py @@ -99,7 +99,7 @@ class BaseTextAggregator(ABC): format. Args: - text: The text to be aggregated + text: The text to be aggregated. Returns: An Aggregation object if ready for processing, or None if more From 3458b74fc958b79cd7b35fb1e882fa298a8ea439 Mon Sep 17 00:00:00 2001 From: Dante Noguez Date: Sat, 22 Nov 2025 09:02:37 -0600 Subject: [PATCH 097/110] Fix 11labs realtime dynamic updates (#3117) --- CHANGELOG.md | 4 ++++ src/pipecat/services/elevenlabs/stt.py | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9208b8d9..24734e799 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Fixed an issue in `ElevenLabsRealtimeSTTService` where dynamic language updates were not working. + ### Added - Added optional speaking rate control to `InworldTTSService`. diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py index 8cbb40d63..03929882e 100644 --- a/src/pipecat/services/elevenlabs/stt.py +++ b/src/pipecat/services/elevenlabs/stt.py @@ -459,6 +459,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): self._audio_format = "" # initialized in start() self._receive_task = None + self._settings = {"language": params.language_code} + def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -477,7 +479,13 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): Changing language requires reconnecting to the WebSocket. """ logger.info(f"Switching STT language to: [{language}]") - self._params.language_code = language.value if isinstance(language, Language) else language + new_language = ( + language_to_elevenlabs_language(language) + if isinstance(language, Language) + else language + ) + self._params.language_code = new_language + self._settings["language"] = new_language # Reconnect with new settings await self._disconnect() await self._connect() From d0ff43134a39cb91a13dff9445a62e405564d1f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 23 Nov 2025 17:42:59 -0800 Subject: [PATCH 098/110] format CHANGELOG --- CHANGELOG.md | 318 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 194 insertions(+), 124 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24734e799..aaaafd512 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,156 +7,235 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Fixed - -- Fixed an issue in `ElevenLabsRealtimeSTTService` where dynamic language updates were not working. - ### Added - Added optional speaking rate control to `InworldTTSService`. -- Introduced a new `AggregatedTextFrame` type to support passing text along with an - `aggregated_by` field to describe the type of text included. `TTSTextFrame`s now - inherit from `AggregatedTextFrame`. With this inheritance, an observer can watch for - `AggregatedTextFrame`s to accumlate the perceived output and determine whether or not - the text was spoken based on if that frame is also a `TTSTextFrame`. +- Introduced a new `AggregatedTextFrame` type to support passing text along with + an `aggregated_by` field to describe the type of text + included. `TTSTextFrame`s now inherit from `AggregatedTextFrame`. With this + inheritance, an observer can watch for `AggregatedTextFrame`s to accumlate the + perceived output and determine whether or not the text was spoken based on if + that frame is also a `TTSTextFrame`. - With this frame, the llm token stream can be transformed into custom composable - chunks, allowing for aggregation outside the TTS service. This makes it possible to - listen for or handle those aggregations and sets the stage for doing things like - composing a best effort of the perceived llm output in a more digestable form and - to do so whether or not it is processed by a TTS or if even a TTS exists. + With this frame, the llm token stream can be transformed into custom + composable chunks, allowing for aggregation outside the TTS service. This + makes it possible to listen for or handle those aggregations and sets the + stage for doing things like composing a best effort of the perceived llm + output in a more digestable form and to do so whether or not it is processed + by a TTS or if even a TTS exists. -- Introduced `LLMTextProcessor`: A new processor meant to allow customization for how - LLMTextFrames should be aggregated and considered. It's purpose is to turn - `LLMTextFrame`s into `AggregatedTextFrame`s. By default, a TTSService will still - aggregate `LLMTextFrame`s by sentence for the service to consume. However, if you - wish to override how the llm text is aggregated, you should no longer override the - TTS's internal text_aggregator, but instead, insert this processor between your LLM - and TTS in the pipeline. +- Introduced `LLMTextProcessor`: A new processor meant to allow customization + for how LLMTextFrames should be aggregated and considered. It's purpose is to + turn `LLMTextFrame`s into `AggregatedTextFrame`s. By default, a TTSService + will still aggregate `LLMTextFrame`s by sentence for the service to + consume. However, if you wish to override how the llm text is aggregated, you + should no longer override the TTS's internal text_aggregator, but instead, + insert this processor between your LLM and TTS in the pipeline. - New `bot-output` RTVI message to represent what the bot actually "says". - - The `RTVIObserver` now emits `bot-output` messages based off the new `AggregatedTextFrame`s - (`bot-tts-text` and `bot-llm-text` are still supported and generated, but `bot-transcript` is - now deprecated in lieu of this new, more thorough, message). + + - The `RTVIObserver` now emits `bot-output` messages based off the new + `AggregatedTextFrame`s (`bot-tts-text` and `bot-llm-text` are still + supported and generated, but `bot-transcript` is now deprecated in lieu of + this new, more thorough, message). + - The new `RTVIBotOutputMessage` includes the fields: + - `spoken`: A boolean indicating whether the text was spoken by TTS - - `aggregated_by`: A string representing how the text was aggregated ("sentence", "word", - "my custom aggregation") - - Introduced new fields to `RTVIObserver` to support the new `bot-output` messaging: - - `bot_output_enabled`: Defaults to True. Set to false to disable bot-output messages. - - `skip_aggregator_types`: Defaults to `None`. Set to a list of strings that match - aggregation types that should not be included in bot-output messages. (Ex. `credit_card`) - - Introduced new methods, `add_text_transformer()` and `remove_text_transformer()`, to - `RTVIObserver` to support providing (and subsequently removing) callbacks for various types of - aggregations (or all aggregations with `*`) that can modify the text before being sent as a - `bot-output` or `tts-text` message. (Think obscuring the credit card or inserting extra detail - the client might want that the context doesn't need.) + + - `aggregated_by`: A string representing how the text was aggregated + ("sentence", "word", "my custom aggregation") + + - Introduced new fields to `RTVIObserver` to support the new `bot-output` + messaging: + + - `bot_output_enabled`: Defaults to True. Set to false to disable bot-output + messages. + + - `skip_aggregator_types`: Defaults to `None`. Set to a list of strings that + match aggregation types that should not be included in bot-output + messages. (Ex. `credit_card`) + + - Introduced new methods, `add_text_transformer()` and + `remove_text_transformer()`, to `RTVIObserver` to support providing (and + subsequently removing) callbacks for various types of aggregations (or all + aggregations with `*`) that can modify the text before being sent as a + `bot-output` or `tts-text` message. (Think obscuring the credit card or + inserting extra detail the client might want that the context doesn't need.) + +- In `MiniMaxHttpTTSService`: + + - Added support for speech-2.6-hd and speech-2.6-turbo models + + - Added languages: Afrikaans, Bulgarian, Catalan, Danish, Persian, Filipino, + Hebrew, Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, + Swedish, and Tamil + + - Added new emotions: calm and fluent ### Changed - Updated `daily-python` to 0.22.0. - `BaseTextAggregator` changes: - Modified the BaseTextAggregator type so that when text gets aggregated, metadata can - be associated with it. Currently, that just means a `type`, so that the aggregation - can be classified or described. Changes made to support this: - - ⚠️ IMPORTANT: Aggregators are now expected to strip leading/trailing white space - characters before returning their aggregation from `aggregation()` or `.text`. This - way all aggregators have a consistent contract allowing downstream use to know how - to stitch aggregations back together. - - Introduced a new `Aggregation` dataclass to represent both the aggregated `text` and - a string identifying the `type` of aggregation (ex. "sentence", "word", "my custom - aggregation") - - ⚠️ Breaking change: `BaseTextAggregator.text` now returns an `Aggregation` (instead of `str`). - To update: `aggregated_text = myAggregator.text` -> `aggregated_text = myAggregator.text.text` - - ⚠️ Breaking change: `BaseTextAggregator.aggregate()` now returns `Optional[Aggregation]` - (instead of `Optional[str]`). To update: - ``` - aggregation = myAggregator.aggregate(text) - if (aggregation): - print(f"successfully aggregated text: {aggregation.text}") // instead of {aggregation} - ``` - - `SimpleTextAggregator`, `SkipTagsAggregator`, `PatternPairAggregator` updated to - produce/consume `Aggregation` objects. + + Modified the BaseTextAggregator type so that when text gets aggregated, + metadata can be associated with it. Currently, that just means a `type`, so + that the aggregation can be classified or described. Changes made to support + this: + + - ⚠️ IMPORTANT: Aggregators are now expected to strip leading/trailing white + space characters before returning their aggregation from `aggregation()` or + `.text`. This way all aggregators have a consistent contract allowing + downstream use to know how to stitch aggregations back together. + + - Introduced a new `Aggregation` dataclass to represent both the aggregated + `text` and a string identifying the `type` of aggregation (ex. "sentence", + "word", "my custom aggregation") + + - ⚠️ Breaking change: `BaseTextAggregator.text` now returns an `Aggregation` + (instead of `str`). + + Before: + + ```python + aggregated_text = myAggregator.text + ``` + + Now: + + ```python + aggregated_text = myAggregator.text.text + ``` + + - ⚠️ Breaking change: `BaseTextAggregator.aggregate()` now returns + `Optional[Aggregation]` (instead of `Optional[str]`). + + Before: + + ```python + aggregation = myAggregator.aggregate(text) + print(f"successfully aggregated text: {aggregation}") + ``` + + Now: + + ```python + aggregation = myAggregator.aggregate(text) + if aggregation: + print(f"successfully aggregated text: {aggregation.text}") + ``` + + - `SimpleTextAggregator`, `SkipTagsAggregator`, `PatternPairAggregator` + updated to produce/consume `Aggregation` objects. + - All uses of the above Aggregators have been updated accordingly. -- Augmented the `PatternPairAggregator` so that matched patterns can be treated as their own - aggregation, taking advantage of the new. To that end: - - Introduced a new, preferred version of `add_pattern` to support a new option for treating a - match as a separate aggregation returned from `aggregate()`. This replaces the now - deprecated `add_pattern_pair` method and you provide a `MatchAction` in lieu of the `remove_match` field. - - `MatchAction` enum: `REMOVE`, `KEEP`, `AGGREGATE`, allowing customization for how - a match should be handled. - - `REMOVE`: The text along with its delimiters will be removed from the streaming text. - Sentence aggregation will continue on as if this text did not exist. - - `KEEP`: The delimiters will be removed, but the content between them will be kept. - Sentence aggregation will continue on with the internal text included. - - `AGGREGATE`: The delimiters will be removed and the content between will be treated - as a separate aggregation. Any text before the start of the pattern will be - returned early, whether or not a complete sentence was found. Then the pattern - will be returned. Then the aggregation will continue on sentence matching after - the closing delimiter is found. The content between the delimiters is not - aggregated by sentence. It is aggregated as one single block of text. - - `PatternMatch` now extends `Aggregation` and provides richer info to handlers. - - ⚠️ Breaking change: The `PatternMatch` type returned to handlers registered via `on_pattern_match` - has been updated to subclass from the new `Aggregation` type, which means that `content` - has been replaced with `text` and `pattern_id` has been replaced with `type`: - ``` - async dev on_match_tag(match: PatternMatch): - pattern = match.type # instead of match.pattern_id - text = match.text # instead of match.content - ``` +- Augmented the `PatternPairAggregator` so that matched patterns can be treated + as their own aggregation, taking advantage of the new. To that end: -- `TextFrame` now includes the field `append_to_context` to support setting whether or not the - encompassing text should be added to the LLM context (by the LLM assistant aggregator). It - defaults to `True`. + - Introduced a new, preferred version of `add_pattern` to support a new option + for treating a match as a separate aggregation returned from + `aggregate()`. This replaces the now deprecated `add_pattern_pair` method + and you provide a `MatchAction` in lieu of the `remove_match` field. + + - `MatchAction` enum: `REMOVE`, `KEEP`, `AGGREGATE`, allowing customization + for how a match should be handled. + + - `REMOVE`: The text along with its delimiters will be removed from the + streaming text. Sentence aggregation will continue on as if this text + did not exist. + + - `KEEP`: The delimiters will be removed, but the content between them + will be kept. Sentence aggregation will continue on with the internal + text included. + + - `AGGREGATE`: The delimiters will be removed and the content between will + be treated as a separate aggregation. Any text before the start of the + pattern will be returned early, whether or not a complete sentence was + found. Then the pattern will be returned. Then the aggregation will + continue on sentence matching after the closing delimiter is found. The + content between the delimiters is not aggregated by sentence. It is + aggregated as one single block of text. + + - `PatternMatch` now extends `Aggregation` and provides richer info to + handlers. + + - ⚠️ Breaking change: The `PatternMatch` type returned to handlers registered + via `on_pattern_match` has been updated to subclass from the new + `Aggregation` type, which means that `content` has been replaced with + `text` and `pattern_id` has been replaced with `type`: + + ```python + async dev on_match_tag(match: PatternMatch): + pattern = match.type # instead of match.pattern_id + text = match.text # instead of match.content + ``` + +- `TextFrame` now includes the field `append_to_context` to support setting + whether or not the encompassing text should be added to the LLM context (by + the LLM assistant aggregator). It defaults to `True`. - `TTSService` base class updates: - - `TTSService`s now accept a new `skip_aggregator_types` to avoid speaking certain aggregation - types (now determined/returned by the aggregator) - - Introduced the ability to do a just-in-time transform of text before it gets sent to the - TTS service via callbacks you can set up via a new init field, `text_transforms` or a new - method `add_text_transformer()`. This makes it possible to do things like introduce - TTS-specific tags for spelling or emotion or change the pronunciation of something on the - fly. `remove_text_transformer` has also been added to support removing a registered - transform callback. - - TTS services push `AggregatedTextFrame` in addition to `TTSTextFrame`s when either an - aggregation occurs that should not be spoken or when the TTS service supports word-by-word - timestamping. In the latter case, the `TTSService` preliminarily generates an - `AggregatedTextFrame`, aggregated by sentence to generate the full sentence content as early - as possible. + + - `TTSService`s now accept a new `skip_aggregator_types` to avoid speaking + certain aggregation types (now determined/returned by the aggregator) + + - Introduced the ability to do a just-in-time transform of text before it gets + sent to the TTS service via callbacks you can set up via a new init field, + `text_transforms` or a new method `add_text_transformer()`. This makes it + possible to do things like introduce TTS-specific tags for spelling or + emotion or change the pronunciation of something on the + fly. `remove_text_transformer` has also been added to support removing a + registered transform callback. + + - TTS services push `AggregatedTextFrame` in addition to `TTSTextFrame`s when + either an aggregation occurs that should not be spoken or when the TTS + service supports word-by-word timestamping. In the latter case, the + `TTSService` preliminarily generates an `AggregatedTextFrame`, aggregated by + sentence to generate the full sentence content as early as possible. - Updated `CartesiaTTSService`: - - Modified use of custom default text_aggregator to avoid deprecation warnings and push users - towards use of transformers or the `LLMTextProcessor` - - Added convenience methods for taking advantage of Cartesia's SSML tags: spell, emotion, - pauses, volume, and speed. + + - Modified use of custom default text_aggregator to avoid deprecation warnings + and push users towards use of transformers or the `LLMTextProcessor` + + - Added convenience methods for taking advantage of Cartesia's SSML tags: + spell, emotion, pauses, volume, and speed. - Updated `RimeTTSService`: - - Modified use of custom default text_aggregator to avoid deprecation warnings and push users - towards use of transformers or the `LLMTextProcessor` - - Added convenience methods for taking advantage of Rime's customization options: spell, - pauses, pronunciations, and inline speed control. + + - Modified use of custom default text_aggregator to avoid deprecation warnings + and push users towards use of transformers or the `LLMTextProcessor` + + - Added convenience methods for taking advantage of Rime's customization + options: spell, pauses, pronunciations, and inline speed control. ### Deprecated - The TTS constructor field, `text_aggregator` is deprecated in favor of the new - `LLMTextProcessor`. TTSServices still have an internal aggregator for support of default - behavior, but if you want to override the aggregation behavior, you should use the new - processor. + `LLMTextProcessor`. TTSServices still have an internal aggregator for support + of default behavior, but if you want to override the aggregation behavior, you + should use the new processor. -- The RTVI `bot-transcription` event is deprecated in favor of the new `bot-output` - message which is the canonical representation of bot output (spoken or not). The code - still emits a transcription message for backwards compatibility while transition occurs. +- The RTVI `bot-transcription` event is deprecated in favor of the new + `bot-output` message which is the canonical representation of bot output + (spoken or not). The code still emits a transcription message for backwards + compatibility while transition occurs. -- Deprecated `add_pattern_pair` in the `PatternPairAggregator` which takes a `pattern_id` - and `remove_match` field in favor of the new `add_pattern` method which takes a `type` and an - `action` +- Deprecated `add_pattern_pair` in the `PatternPairAggregator` which takes a + `pattern_id` and `remove_match` field in favor of the new `add_pattern` method + which takes a `type` and an `action` +- `english_normalization` input parameter for `MiniMaxHttpTTSService` is + deprecated, use `test_normalization` instead. ### Fixed +- Fixed an issue in `ElevenLabsRealtimeSTTService` where dynamic language + updates were not working. + - Fixed `InworldTTSService` audio config payload to use camelCase keys expected by the Inworld API. @@ -218,20 +297,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated language mappings for the Google and Gemini TTS services to match official documentation. -- In `MiniMaxHttpTTSService`: --- Added support for speech-2.6-hd and speech-2.6-turbo models --- Added languages: Afrikaans, Bulgarian, Catalan, Danish, Persian, Filipino, Hebrew, -Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and Tamil --- Added new emotions: calm and fluent - ### Deprecated - The `api_key` parameter in `GeminiTTSService` is deprecated. Use `credentials` or `credentials_path` instead for Google Cloud authentication. -- `english_normalization` input parameter for `MiniMaxHttpTTSService` is deprecated, -use `test_normalization` instead. - ### Fixed - Fixed a `SimliVideoService` connection issue. From 7bdac028375098fe06c2bbd51f43ffd390e86497 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Nov 2025 14:21:58 -0500 Subject: [PATCH 099/110] Fix sample_rate issue in ElevenLabsRealtimeSTTService, add timestamps and logging --- CHANGELOG.md | 37 +++++++++---- src/pipecat/services/elevenlabs/stt.py | 77 +++++++++++++++++++++----- 2 files changed, 89 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aaaafd512..a40711642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for `include_timestamps` and `enable_logging` in + `ElevenLabsRealtimeSTTService`. When `include_timestamps` is enabled, + timestamp data is included in the `TranscriptionFrame`'s `result` + parameter. + - Added optional speaking rate control to `InworldTTSService`. - Introduced a new `AggregatedTextFrame` type to support passing text along with @@ -144,11 +149,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 for how a match should be handled. - `REMOVE`: The text along with its delimiters will be removed from the - streaming text. Sentence aggregation will continue on as if this text + streaming text. Sentence aggregation will continue on as if this text did not exist. - `KEEP`: The delimiters will be removed, but the content between them - will be kept. Sentence aggregation will continue on with the internal + will be kept. Sentence aggregation will continue on with the internal text included. - `AGGREGATE`: The delimiters will be removed and the content between will @@ -163,15 +168,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 handlers. - ⚠️ Breaking change: The `PatternMatch` type returned to handlers registered - via `on_pattern_match` has been updated to subclass from the new - `Aggregation` type, which means that `content` has been replaced with - `text` and `pattern_id` has been replaced with `type`: + via `on_pattern_match` has been updated to subclass from the new + `Aggregation` type, which means that `content` has been replaced with + `text` and `pattern_id` has been replaced with `type`: - ```python - async dev on_match_tag(match: PatternMatch): - pattern = match.type # instead of match.pattern_id - text = match.text # instead of match.content - ``` + ```python + async dev on_match_tag(match: PatternMatch): + pattern = match.type # instead of match.pattern_id + text = match.text # instead of match.content + ``` - `TextFrame` now includes the field `append_to_context` to support setting whether or not the encompassing text should be added to the LLM context (by @@ -236,6 +241,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue in `ElevenLabsRealtimeSTTService` where dynamic language updates were not working. +- Fixed an issue in `ElevenLabsRealtimeSTTService` where setting the sample + rate would result in transcripts failing. + - Fixed `InworldTTSService` audio config payload to use camelCase keys expected by the Inworld API. @@ -297,11 +305,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated language mappings for the Google and Gemini TTS services to match official documentation. +- In `MiniMaxHttpTTSService`: + -- Added support for speech-2.6-hd and speech-2.6-turbo models + -- Added languages: Afrikaans, Bulgarian, Catalan, Danish, Persian, Filipino, Hebrew, + Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and Tamil + -- Added new emotions: calm and fluent + ### Deprecated - The `api_key` parameter in `GeminiTTSService` is deprecated. Use `credentials` or `credentials_path` instead for Google Cloud authentication. +- `english_normalization` input parameter for `MiniMaxHttpTTSService` is deprecated, + use `test_normalization` instead. + ### Fixed - Fixed a `SimliVideoService` connection issue. diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py index 03929882e..95a802edd 100644 --- a/src/pipecat/services/elevenlabs/stt.py +++ b/src/pipecat/services/elevenlabs/stt.py @@ -416,6 +416,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): Only used when commit_strategy is VAD. None uses ElevenLabs default. min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms). Only used when commit_strategy is VAD. None uses ElevenLabs default. + include_timestamps: Whether to include word-level timestamps in transcripts. + enable_logging: Whether to enable logging on ElevenLabs' side. """ language_code: Optional[str] = None @@ -424,6 +426,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): vad_threshold: Optional[float] = None min_speech_duration_ms: Optional[int] = None min_silence_duration_ms: Optional[int] = None + include_timestamps: bool = False + enable_logging: bool = False def __init__( self, @@ -628,10 +632,16 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): if self._params.language_code: params.append(f"language_code={self._params.language_code}") - params.append(f"encoding={self._audio_format}") - params.append(f"sample_rate={self.sample_rate}") + params.append(f"audio_format={self._audio_format}") params.append(f"commit_strategy={self._params.commit_strategy.value}") + # Add optional parameters + if self._params.include_timestamps: + params.append(f"include_timestamps={str(self._params.include_timestamps).lower()}") + + if self._params.enable_logging: + params.append(f"enable_logging={str(self._params.enable_logging).lower()}") + # Add VAD parameters if using VAD commit strategy and values are specified if self._params.commit_strategy == CommitStrategy.VAD: if self._params.vad_silence_threshold_secs is not None: @@ -720,15 +730,20 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): elif message_type == "committed_transcript_with_timestamps": await self._on_committed_transcript_with_timestamps(data) - elif message_type == "input_error": - error_msg = data.get("error", "Unknown input error") - logger.error(f"ElevenLabs input error: {error_msg}") - await self.push_error(ErrorFrame(f"Input error: {error_msg}")) + elif message_type == "error": + error_msg = data.get("error", "Unknown error") + logger.error(f"ElevenLabs error: {error_msg}") + await self.push_error(ErrorFrame(f"Error: {error_msg}")) - elif message_type in ["auth_error", "quota_exceeded", "transcriber_error", "error"]: - error_msg = data.get("error", data.get("message", "Unknown error")) - logger.error(f"ElevenLabs error ({message_type}): {error_msg}") - await self.push_error(ErrorFrame(f"{message_type}: {error_msg}")) + elif message_type == "auth_error": + error_msg = data.get("error", "Authentication error") + logger.error(f"ElevenLabs auth error: {error_msg}") + await self.push_error(ErrorFrame(f"Auth error: {error_msg}")) + + elif message_type == "quota_exceeded_error": + error_msg = data.get("error", "Quota exceeded") + logger.error(f"ElevenLabs quota exceeded: {error_msg}") + await self.push_error(ErrorFrame(f"Quota exceeded: {error_msg}")) else: logger.debug(f"Unknown message type: {message_type}") @@ -773,6 +788,11 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): Args: data: Committed transcript data. """ + # If timestamps are enabled, skip this message and wait for the + # committed_transcript_with_timestamps message which contains all the data + if self._params.include_timestamps: + return + text = data.get("text", "").strip() if not text: return @@ -800,6 +820,18 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): async def _on_committed_transcript_with_timestamps(self, data: dict): """Handle committed transcript with word-level timestamps. + This message is sent when include_timestamps=true. The result data includes: + - text: The transcribed text + - language_code: Detected language (if available) + - words: Array of word objects with timing information: + - text: The word text + - start: Start time in seconds + - end: End time in seconds + - type: "word" or "spacing" + - speaker_id: Speaker identifier (if available) + - logprob: Log probability score (if available) + - characters: Array of character strings (if available) + Args: data: Committed transcript data with timestamps. """ @@ -807,9 +839,24 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): if not text: return - logger.debug(f"Committed transcript with timestamps: [{text}]") - logger.trace(f"Timestamps: {data.get('words', [])}") + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() - # This is sent after the committed_transcript, so we don't need to - # push another TranscriptionFrame, but we could use the timestamps - # for additional processing if needed in the future + # Get language if provided + language = data.get("language_code") + + logger.debug(f"Committed transcript with timestamps: [{text}]") + + await self._handle_transcription(text, True, language) + + # This message is sent after committed_transcript when include_timestamps=true. + # It contains the full transcript data including text and word-level timestamps. + await self.push_frame( + TranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + language, + result=data, + ) + ) From 87fdd8f006f24ce6c1ab669cf5843aae9a3f3961 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Nov 2025 14:24:27 -0500 Subject: [PATCH 100/110] Fix MiniMax changelog entries --- CHANGELOG.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a40711642..699a946d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -305,20 +305,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated language mappings for the Google and Gemini TTS services to match official documentation. -- In `MiniMaxHttpTTSService`: - -- Added support for speech-2.6-hd and speech-2.6-turbo models - -- Added languages: Afrikaans, Bulgarian, Catalan, Danish, Persian, Filipino, Hebrew, - Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and Tamil - -- Added new emotions: calm and fluent - ### Deprecated - The `api_key` parameter in `GeminiTTSService` is deprecated. Use `credentials` or `credentials_path` instead for Google Cloud authentication. -- `english_normalization` input parameter for `MiniMaxHttpTTSService` is deprecated, - use `test_normalization` instead. - ### Fixed - Fixed a `SimliVideoService` connection issue. From ab8dcd6edead6686f07ba9d7a6476f07310f4fb2 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 22 Nov 2025 07:07:21 -0500 Subject: [PATCH 101/110] Add SageMaker BiDi client --- CHANGELOG.md | 3 + pyproject.toml | 3 +- src/pipecat/services/aws/__init__.py | 1 + .../services/aws/sagemaker/__init__.py | 0 .../services/aws/sagemaker/bidi_client.py | 283 ++++++++++++++++++ uv.lock | 86 +++--- 6 files changed, 341 insertions(+), 35 deletions(-) create mode 100644 src/pipecat/services/aws/sagemaker/__init__.py create mode 100644 src/pipecat/services/aws/sagemaker/bidi_client.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 699a946d0..3a767a777 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `SageMakerBidiClient` to connect to SageMaker hosted BiDi compatible + services. + - Added support for `include_timestamps` and `enable_logging` in `ElevenLabsRealtimeSTTService`. When `include_timestamps` is enabled, timestamp data is included in the `TranscriptionFrame`'s `result` diff --git a/pyproject.toml b/pyproject.toml index 73da9083a..e4b0a380b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] aws = [ "aioboto3~=15.0.0", "pipecat-ai[websockets-base]" ] -aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.1.1; python_version>='3.12'" ] +aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.2.0; python_version>='3.12'" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] cerebras = [] @@ -98,6 +98,7 @@ sentry = [ "sentry-sdk>=2.28.0,<3" ] local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "torchaudio>=2.5.0,<3" ] local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] remote-smart-turn = [] +sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"] silero = [ "onnxruntime>=1.20.1,<2" ] simli = [ "simli-ai~=1.0.3"] soniox = [ "pipecat-ai[websockets-base]" ] diff --git a/src/pipecat/services/aws/__init__.py b/src/pipecat/services/aws/__init__.py index 3cdd4cc5a..6f6903f75 100644 --- a/src/pipecat/services/aws/__init__.py +++ b/src/pipecat/services/aws/__init__.py @@ -10,6 +10,7 @@ from pipecat.services import DeprecatedModuleProxy from .llm import * from .nova_sonic import * +from .sagemaker import * from .stt import * from .tts import * diff --git a/src/pipecat/services/aws/sagemaker/__init__.py b/src/pipecat/services/aws/sagemaker/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/services/aws/sagemaker/bidi_client.py b/src/pipecat/services/aws/sagemaker/bidi_client.py new file mode 100644 index 000000000..5e02af03d --- /dev/null +++ b/src/pipecat/services/aws/sagemaker/bidi_client.py @@ -0,0 +1,283 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""AWS SageMaker bidirectional streaming client. + +This module provides a client for streaming bidirectional communication with +SageMaker endpoints using the HTTP/2 protocol. Supports sending audio, text, +and JSON data to SageMaker model endpoints and receiving streaming responses. +""" + +import os +from typing import Optional + +from loguru import logger + +try: + from aws_sdk_sagemaker_runtime_http2.client import SageMakerRuntimeHTTP2Client + from aws_sdk_sagemaker_runtime_http2.config import Config, HTTPAuthSchemeResolver + from aws_sdk_sagemaker_runtime_http2.models import ( + InvokeEndpointWithBidirectionalStreamInput, + RequestPayloadPart, + RequestStreamEventPayloadPart, + ResponseStreamEvent, + ) + from smithy_aws_core.auth.sigv4 import SigV4AuthScheme + from smithy_aws_core.identity import EnvironmentCredentialsResolver + from smithy_core.aio.eventstream import DuplexEventStream +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use SageMaker BiDi client, you need to `pip install pipecat-ai[sagemaker]`." + ) + raise Exception(f"Missing module: {e}") + + +class SageMakerBidiClient: + """Client for bidirectional streaming with AWS SageMaker endpoints. + + Handles low-level HTTP/2 bidirectional streaming protocol for communicating + with SageMaker model endpoints. Provides methods for sending various data + types (audio, text, JSON) and receiving streaming responses. + + This client uses AWS SigV4 authentication and supports credential resolution + from environment variables, AWS CLI configuration, and instance metadata. + + Example:: + + client = SageMakerBidiClient( + endpoint_name="my-deepgram-endpoint", + region="us-east-2", + model_invocation_path="v1/listen", + model_query_string="model=nova-3&language=en" + ) + await client.start_session() + await client.send_audio_chunk(audio_bytes) + response = await client.receive_response() + await client.close_session() + """ + + def __init__( + self, + endpoint_name: str, + region: str, + model_invocation_path: str = "", + model_query_string: str = "", + ): + """Initialize the SageMaker BiDi client. + + Args: + endpoint_name: Name of the SageMaker endpoint to connect to. + region: AWS region where the endpoint is deployed. + model_invocation_path: API path for the model invocation (e.g., "v1/listen"). + model_query_string: Query string parameters for the model (e.g., "model=nova-3"). + """ + self.endpoint_name = endpoint_name + self.region = region + self.model_invocation_path = model_invocation_path + self.model_query_string = model_query_string + self.bidi_endpoint = f"https://runtime.sagemaker.{region}.amazonaws.com:8443" + self._client: Optional[SageMakerRuntimeHTTP2Client] = None + self._stream: Optional[ + DuplexEventStream[RequestStreamEventPayloadPart, ResponseStreamEvent, any] + ] = None + self._output_stream = None + self._is_active = False + + def _initialize_client(self): + """Initialize the SageMaker Runtime HTTP2 client with AWS credentials. + + Creates and configures the SageMaker Runtime HTTP2 client with SigV4 + authentication. Attempts to resolve AWS credentials from environment + variables, AWS CLI configuration, or instance metadata. + """ + logger.debug(f"Initializing SageMaker BiDi client for region: {self.region}") + logger.debug(f"Using endpoint URI: {self.bidi_endpoint}") + + # Check for AWS credentials + has_env_creds = bool(os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY")) + + if not has_env_creds: + logger.warning( + "AWS credentials not found in environment variables. " + "Attempting to use EnvironmentCredentialsResolver which will check " + "AWS CLI configuration and instance metadata." + ) + + config = Config( + endpoint_uri=self.bidi_endpoint, + region=self.region, + aws_credentials_identity_resolver=EnvironmentCredentialsResolver(), + auth_scheme_resolver=HTTPAuthSchemeResolver(), + auth_schemes={"aws.auth#sigv4": SigV4AuthScheme(service="sagemaker")}, + ) + self._client = SageMakerRuntimeHTTP2Client(config=config) + + async def start_session(self): + """Start a bidirectional streaming session with the SageMaker endpoint. + + Initializes the client if needed, creates the bidirectional stream, and + establishes the connection to the SageMaker endpoint. Must be called + before sending or receiving data. + + Returns: + The output stream for receiving responses. + + Raises: + RuntimeError: If client initialization or connection fails. + """ + if not self._client: + self._initialize_client() + + logger.debug(f"Starting BiDi session with endpoint: {self.endpoint_name}") + logger.debug(f"Model invocation path: {self.model_invocation_path}") + logger.debug(f"Model query string: {self.model_query_string}") + + # Create the bidirectional stream + stream_input = InvokeEndpointWithBidirectionalStreamInput( + endpoint_name=self.endpoint_name, + model_invocation_path=self.model_invocation_path, + model_query_string=self.model_query_string, + ) + + try: + self._stream = await self._client.invoke_endpoint_with_bidirectional_stream( + stream_input + ) + self._is_active = True + + # Get output stream + output = await self._stream.await_output() + self._output_stream = output[1] + + logger.debug("BiDi session started successfully") + return self._output_stream + + except Exception as e: + logger.error(f"Failed to start BiDi session: {e}") + self._is_active = False + raise RuntimeError(f"Failed to start SageMaker BiDi session: {e}") + + async def send_data(self, data_bytes: bytes, data_type: Optional[str] = None): + """Send a chunk of data to the stream. + + Generic method for sending any type of data to the SageMaker endpoint. + Use the convenience methods (send_audio_chunk, send_text, send_json) + for common data types. + + Args: + data_bytes: Raw bytes to send. + data_type: Optional data type header. Common values are "BINARY" for + audio/binary data and "UTF8" for text/JSON data. + + Raises: + RuntimeError: If session is not active or send fails. + """ + if not self._is_active or not self._stream: + raise RuntimeError("BiDi session not active") + + try: + payload = RequestPayloadPart(bytes_=data_bytes, data_type=data_type) + event = RequestStreamEventPayloadPart(value=payload) + await self._stream.input_stream.send(event) + except Exception as e: + logger.error(f"Failed to send data: {e}") + raise + + async def send_audio_chunk(self, audio_bytes: bytes): + """Send a chunk of audio data to the stream. + + Convenience method for sending audio data. Automatically sets the data + type to "BINARY". + + Args: + audio_bytes: Raw audio bytes to send (e.g., PCM audio data). + + Raises: + RuntimeError: If session is not active or send fails. + """ + await self.send_data(audio_bytes, data_type="BINARY") + + async def send_text(self, text: str): + """Send text data to the stream. + + Convenience method for sending text data. Automatically encodes the text + as UTF-8 and sets the data type to "UTF8". + + Args: + text: Text string to send. + + Raises: + RuntimeError: If session is not active or send fails. + """ + await self.send_data(text.encode("utf-8"), data_type="UTF8") + + async def send_json(self, data: dict): + """Send JSON data to the stream. + + Convenience method for sending JSON-encoded messages. Useful for control + messages like KeepAlive or CloseStream. Automatically serializes the + dictionary to JSON, encodes as UTF-8, and sets the data type to "UTF8". + + Args: + data: Dictionary to send as JSON (e.g., {"type": "KeepAlive"}). + + Raises: + RuntimeError: If session is not active or send fails. + """ + import json + + await self.send_data(json.dumps(data).encode("utf-8"), data_type="UTF8") + + async def receive_response(self) -> Optional[ResponseStreamEvent]: + """Receive a response from the stream. + + Blocks until a response is available from the SageMaker endpoint. Returns + None when the stream is closed. + + Returns: + The response event containing payload data, or None if stream is closed. + + Raises: + RuntimeError: If session is not active. + """ + if not self._is_active or not self._output_stream: + raise RuntimeError("BiDi session not active") + + try: + result = await self._output_stream.receive() + return result + except Exception as e: + logger.error(f"Failed to receive response: {e}") + raise + + async def close_session(self): + """Close the bidirectional streaming session. + + Gracefully closes the input stream and marks the session as inactive. + Safe to call multiple times. + """ + if not self._is_active: + return + + logger.debug("Closing BiDi session...") + self._is_active = False + + try: + if self._stream: + await self._stream.input_stream.close() + logger.debug("BiDi session closed successfully") + except Exception as e: + logger.warning(f"Error closing BiDi session: {e}") + + @property + def is_active(self) -> bool: + """Check if the session is currently active. + + Returns: + True if session is active, False otherwise. + """ + return self._is_active diff --git a/uv.lock b/uv.lock index 662150b5b..eb2fca39c 100644 --- a/uv.lock +++ b/uv.lock @@ -419,16 +419,30 @@ wheels = [ [[package]] name = "aws-sdk-bedrock-runtime" -version = "0.1.1" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "smithy-aws-core", extra = ["eventstream", "json"], marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, { name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1d/78/48574454b3cac869df67665e4a403ebfc3abfcfba2c2ff01ccfd67d55f8f/aws_sdk_bedrock_runtime-0.1.1.tar.gz", hash = "sha256:c896f99e675c3a1ab600633a07b785f3dc9fe8ab94f640b1f992b63da2dfc784", size = 82446, upload-time = "2025-10-21T20:25:25.845Z" } +sdist = { url = "https://files.pythonhosted.org/packages/db/94/f2451bb09c106e5690bbb88fc366637cdcec942b352ed9bb788804c877e0/aws_sdk_bedrock_runtime-0.2.0.tar.gz", hash = "sha256:8de52dd4492e74c73244d4b41a52304e1db368814a10e49dbbf8f4e8e412cd0e", size = 88156, upload-time = "2025-11-22T00:35:44.978Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/83/07/62c0b70223d178c138f29124ac2f7973a6ba803abc7735b6a01a85217f3d/aws_sdk_bedrock_runtime-0.1.1-py3-none-any.whl", hash = "sha256:c0336b377b2112cf88197d3d44302fbeb3efb1101989fa49ae55e78f49cfe345", size = 74954, upload-time = "2025-10-21T20:25:24.973Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6b/07fbddd31dd6e38c967fe088b5e91a7cc3a2bc0f645f18b4e5d45bc03f1f/aws_sdk_bedrock_runtime-0.2.0-py3-none-any.whl", hash = "sha256:19594de50a52d199d73efca153c0a2328bd781827715a6e012d50b11085236cc", size = 79875, upload-time = "2025-11-22T00:35:44.092Z" }, +] + +[[package]] +name = "aws-sdk-sagemaker-runtime-http2" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smithy-aws-core", extra = ["eventstream", "json"], marker = "python_full_version >= '3.12'" }, + { name = "smithy-core", marker = "python_full_version >= '3.12'" }, + { name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/ca/00f9c55887fc0f3fa345995dd871d40ff81473ab1591e56b4b4483d99d00/aws_sdk_sagemaker_runtime_http2-0.1.0.tar.gz", hash = "sha256:5077ec0c4440495b15004bbf04e27bc0bc137f1f8950d32195c6b45d7788d837", size = 20863, upload-time = "2025-11-22T00:20:56.358Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/24/2e2f727c51c20f4625cd19364d9421dbd7c893fe2b53a46eb0caaf6263a2/aws_sdk_sagemaker_runtime_http2-0.1.0-py3-none-any.whl", hash = "sha256:1aebb728ba6c6d14e58e29ecf89b51f7abbe8786d34144f8a7d59a419e80bd2f", size = 21911, upload-time = "2025-11-22T00:20:55.054Z" }, ] [[package]] @@ -4569,6 +4583,9 @@ runner = [ { name = "python-dotenv" }, { name = "uvicorn" }, ] +sagemaker = [ + { name = "aws-sdk-sagemaker-runtime-http2", marker = "python_full_version >= '3.12'" }, +] sarvam = [ { name = "sarvamai" }, { name = "websockets" }, @@ -4654,7 +4671,8 @@ requires-dist = [ { name = "aiortc", marker = "extra == 'webrtc'", specifier = ">=1.13.0,<2" }, { name = "anthropic", marker = "extra == 'anthropic'", specifier = "~=0.49.0" }, { name = "audioop-lts", marker = "python_full_version >= '3.13'", specifier = "~=0.2.1" }, - { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'aws-nova-sonic'", specifier = "~=0.1.1" }, + { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'aws-nova-sonic'", specifier = "~=0.2.0" }, + { name = "aws-sdk-sagemaker-runtime-http2", marker = "python_full_version >= '3.12' and extra == 'sagemaker'" }, { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.42.0" }, { name = "cartesia", marker = "extra == 'cartesia'", specifier = "~=2.0.3" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" }, @@ -4745,7 +4763,7 @@ requires-dist = [ { name = "wait-for2", marker = "python_full_version < '3.12'", specifier = ">=0.4.1" }, { name = "websockets", marker = "extra == 'websockets-base'", specifier = ">=13.1,<16.0" }, ] -provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "cerebras", "deepseek", "daily", "deepgram", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "krisp", "koala", "langchain", "livekit", "lmnt", "local", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "nim", "neuphonic", "noisereduce", "openai", "openpipe", "openrouter", "perplexity", "playht", "qwen", "rime", "riva", "runner", "sambanova", "sarvam", "sentry", "local-smart-turn", "local-smart-turn-v3", "remote-smart-turn", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] +provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "cerebras", "deepseek", "daily", "deepgram", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "krisp", "koala", "langchain", "livekit", "lmnt", "local", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "nim", "neuphonic", "noisereduce", "openai", "openpipe", "openrouter", "perplexity", "playht", "qwen", "rime", "riva", "runner", "sambanova", "sarvam", "sentry", "local-smart-turn", "local-smart-turn-v3", "remote-smart-turn", "sagemaker", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] [package.metadata.requires-dev] dev = [ @@ -6522,16 +6540,16 @@ wheels = [ [[package]] name = "smithy-aws-core" -version = "0.1.1" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aws-sdk-signers", marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, { name = "smithy-http", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/d3/f847e0fd36b95aa36ce3a4c9ce1a08e16b2aa9a56b71714045c9c924e282/smithy_aws_core-0.1.1.tar.gz", hash = "sha256:78dfd7040fc2bc72b6af293096642fc9a7bfd2db28ddbdf7c4110535eab9d662", size = 11196, upload-time = "2025-10-21T20:21:18.648Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c1/c8/5970c869527972b23a1733de3993d54283c84a2340e84acdd48a11aa0ff4/smithy_aws_core-0.2.0.tar.gz", hash = "sha256:dfa1ecd311d6f0a16f48c86d793085e2a0a33a46de897d129dd1f142a43bf7f6", size = 11344, upload-time = "2025-11-21T18:33:01.928Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/04/87cb06f0f6d664b5cffdef6d4042dd52c11c138436084d30ffdaa3543031/smithy_aws_core-0.1.1-py3-none-any.whl", hash = "sha256:0d1634f276c2999dc2a04fafef63b9d28309de50d939d1d49df952773a7063c4", size = 18963, upload-time = "2025-10-21T20:21:17.692Z" }, + { url = "https://files.pythonhosted.org/packages/88/25/739c0005a6cb4effbc2d37fe23590660b508fe314200f4acf94410a8f315/smithy_aws_core-0.2.0-py3-none-any.whl", hash = "sha256:d112082ef77758e1977f8694cf690ac35c76570c12a6590fccd5da085a3ce507", size = 18966, upload-time = "2025-11-21T18:33:00.812Z" }, ] [package.optional-dependencies] @@ -6544,35 +6562,35 @@ json = [ [[package]] name = "smithy-aws-event-stream" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "smithy-core", marker = "python_full_version >= '3.12'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/26/8ff24194efed60b2df18f610ea05fa2a4c6546858b80a0a51335a4943b9b/smithy_aws_event_stream-0.1.0.tar.gz", hash = "sha256:6634691a3bf5d4801a2c29f0761db2dc4771f3ae43cdee50c10d4b4bb2f86475", size = 12216, upload-time = "2025-09-29T19:37:14.659Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/c4/2b63d31af58fc359577e5515bf730348a235f2f2fa10e17af8640495c81c/smithy_aws_event_stream-0.1.0-py3-none-any.whl", hash = "sha256:17a7300a85cb90df4c6c23f895ca6343361fa419203c3cf80019edd7d3b5f036", size = 15581, upload-time = "2025-09-29T19:37:13.589Z" }, -] - -[[package]] -name = "smithy-core" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/8d/16028d03456071d21de7591f1e1e6a1cc81b2389e53ef8663dbf59caf9cd/smithy_core-0.1.0.tar.gz", hash = "sha256:b159b8905264e1e4c613eab9f74cec0b2f5b8119c42fbadddb4da0a8ed8050e9", size = 48415, upload-time = "2025-09-29T19:37:16.873Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/5b/563cb2beadcfa40597b0c3ff3f2d42e21f065b14782c4ba9cb41a44b745f/smithy_core-0.1.0-py3-none-any.whl", hash = "sha256:cb44e9355fb89e89f2c6ba6a1d59c5db4f2f7282c72d31d9307b6202d66cd0fa", size = 62895, upload-time = "2025-09-29T19:37:15.917Z" }, -] - -[[package]] -name = "smithy-http" version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "smithy-core", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3c/1c/44e99a7dfb8c39bf0c3d998accdf4573a7a3488863b90f21af260cec2d45/smithy_http-0.2.0.tar.gz", hash = "sha256:2382562fa9af326be455f14b18615f16ffe9db756e51b2a4ca0d23e1b881cff8", size = 26729, upload-time = "2025-10-21T20:21:06.146Z" } +sdist = { url = "https://files.pythonhosted.org/packages/31/90/78283c21484f8cf9862982e53bc2769b784910735fb5fb2400a17bfb5fdd/smithy_aws_event_stream-0.2.0.tar.gz", hash = "sha256:99700a11346e7ab1435ff2e53e6f6d60a1e857f2b2ee1941d40b54270adf3323", size = 12278, upload-time = "2025-11-21T18:33:03.79Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/e2/d475fad81ac74ec0e145cb6d72afe5ecde4e2358bd632c2fd5d3f4bc87dc/smithy_http-0.2.0-py3-none-any.whl", hash = "sha256:49ee2402d7737798d70f99f491fbfb2a5767283ae562e21b6f86e3fd14f3e3e0", size = 37328, upload-time = "2025-10-21T20:21:05.362Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/08b997eee81b55150496ce565f0e03c72d0c80e5b218170bdeae7c46a5a4/smithy_aws_event_stream-0.2.0-py3-none-any.whl", hash = "sha256:679a0c7d944e67d3a55d287541b3ca1e61f9d6a62e13401367dcc034e75aa55d", size = 15567, upload-time = "2025-11-21T18:33:02.711Z" }, +] + +[[package]] +name = "smithy-core" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/f6/140f0be9331dd7cd8fa012b3ca4735df39a1a81d03eea89728f997249116/smithy_core-0.2.0.tar.gz", hash = "sha256:05c3e3309df5dcb9cf53e241bd57a96510e4575186443ea157db9dbb59b6c85e", size = 50334, upload-time = "2025-11-21T18:33:05.697Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/e3/d0defa2acf50b91625fe15e3ddb0c8e41ff64363a1f4cd9b8f19ae2ec0c6/smithy_core-0.2.0-py3-none-any.whl", hash = "sha256:db4620da3497abb60f79ac1d8a738d3eac46d7e820bfb50c777c36e932915239", size = 64777, upload-time = "2025-11-21T18:33:04.591Z" }, +] + +[[package]] +name = "smithy-http" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smithy-core", marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/c7/4d8be56e897f99f3b6ffcdf52ba00a468febc939fca85b90f1c122450830/smithy_http-0.3.0.tar.gz", hash = "sha256:55dcc3af315eee6863d2f3f58ada1d9cb4bcc3a57faac10a1b21d4a93722f520", size = 28674, upload-time = "2025-11-21T18:33:07.387Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/e5/59ae79ecdc9a935ad10512c581b3054ebb1afd90498ecc8afaf141dbc22b/smithy_http-0.3.0-py3-none-any.whl", hash = "sha256:972924304febd77c7134a7cffab83ce3b48423ff966dcc1f257e2c0d58fa9b18", size = 40520, upload-time = "2025-11-21T18:33:06.312Z" }, ] [package.optional-dependencies] @@ -6582,15 +6600,15 @@ awscrt = [ [[package]] name = "smithy-json" -version = "0.1.0" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ijson", marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e2/5b/0ecb10007475e1b8faca3bbff1be2fc6edb3ea12ffc5e939e2249be95325/smithy_json-0.1.0.tar.gz", hash = "sha256:84fb48e445b87d850c240d837702c16b259ea53bad76b655ac1bbd8094d48912", size = 7086, upload-time = "2025-09-29T19:37:20.432Z" } +sdist = { url = "https://files.pythonhosted.org/packages/89/cf/e319a2a299b27bc0addf46ee3d4b9c25ec0817e3a0507b2b7a33eddc19f1/smithy_json-0.2.0.tar.gz", hash = "sha256:0946066fdda15d6a579dfdd4b61a547ab915eb057bd176fc2bc17d01dc789499", size = 7157, upload-time = "2025-11-21T18:33:08.968Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/95/e11c04e56aae12b62e38c49000004a1dc598a64dc207018c08448efde322/smithy_json-0.1.0-py3-none-any.whl", hash = "sha256:80ff64734dccdabf1ba6a2908555b97e60f62c07c3a27df48e421ee058413cb9", size = 9914, upload-time = "2025-09-29T19:37:19.459Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b1/33012ac5b2e5940a00b6e1ccc313330e6f8692152a151f72a398cd6be0e0/smithy_json-0.2.0-py3-none-any.whl", hash = "sha256:5018a4e61731afa3094a02d737d4f956dbf270c271410c089045a17d86fc3b3b", size = 9911, upload-time = "2025-11-21T18:33:08.267Z" }, ] [[package]] From 782b257bbb2b6cf9f77fff6e6a8ab243ff8bd235 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 22 Nov 2025 07:09:54 -0500 Subject: [PATCH 102/110] Add DeepgramSageMakerSTTService --- CHANGELOG.md | 3 + src/pipecat/services/deepgram/__init__.py | 1 + .../services/deepgram/stt_sagemaker.py | 447 ++++++++++++++++++ 3 files changed, 451 insertions(+) create mode 100644 src/pipecat/services/deepgram/stt_sagemaker.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a767a777..93e0ac4dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `DeepgramSageMakerSTTService` which connects to a SageMaker hosted + Deepgram STT model. + - Added `SageMakerBidiClient` to connect to SageMaker hosted BiDi compatible services. diff --git a/src/pipecat/services/deepgram/__init__.py b/src/pipecat/services/deepgram/__init__.py index c23ebbec9..227ac5c64 100644 --- a/src/pipecat/services/deepgram/__init__.py +++ b/src/pipecat/services/deepgram/__init__.py @@ -10,6 +10,7 @@ from pipecat.services import DeprecatedModuleProxy from .flux import * from .stt import * +from .stt_sagemaker import * from .tts import * sys.modules[__name__] = DeprecatedModuleProxy(globals(), "deepgram", "deepgram.[stt,tts]") diff --git a/src/pipecat/services/deepgram/stt_sagemaker.py b/src/pipecat/services/deepgram/stt_sagemaker.py new file mode 100644 index 000000000..6d28feefa --- /dev/null +++ b/src/pipecat/services/deepgram/stt_sagemaker.py @@ -0,0 +1,447 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Deepgram speech-to-text service for AWS SageMaker. + +This module provides a Pipecat STT service that connects to Deepgram models +deployed on AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for +low-latency real-time transcription with support for interim results, multiple +languages, and various Deepgram features. +""" + +import asyncio +import json +from typing import AsyncGenerator, Optional + +from loguru import logger + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + InterimTranscriptionFrame, + StartFrame, + TranscriptionFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient +from pipecat.services.stt_service import STTService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_stt + +try: + from deepgram import LiveOptions +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use DeepgramSageMakerSTTService, you need to `pip install pipecat-ai[deepgram,sagemaker]`." + ) + raise Exception(f"Missing module: {e}") + + +class DeepgramSageMakerSTTService(STTService): + """Deepgram speech-to-text service for AWS SageMaker. + + Provides real-time speech recognition using Deepgram models deployed on + AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for low-latency + transcription with support for interim results, speaker diarization, and + multiple languages. + + Requirements: + + - AWS credentials configured (via environment variables, AWS CLI, or instance metadata) + - A deployed SageMaker endpoint with Deepgram model: https://developers.deepgram.com/docs/deploy-amazon-sagemaker + - Deepgram SDK for LiveOptions configuration + + Example:: + + stt = DeepgramSageMakerSTTService( + endpoint_name="my-deepgram-endpoint", + region="us-east-2", + live_options=LiveOptions( + model="nova-3", + language="en", + interim_results=True, + punctuate=True, + ), + ) + """ + + def __init__( + self, + *, + endpoint_name: str, + region: str, + sample_rate: Optional[int] = None, + live_options: Optional[LiveOptions] = None, + **kwargs, + ): + """Initialize the Deepgram SageMaker STT service. + + Args: + endpoint_name: Name of the SageMaker endpoint with Deepgram model + deployed (e.g., "my-deepgram-nova-3-endpoint"). + region: AWS region where the endpoint is deployed (e.g., "us-east-2"). + sample_rate: Audio sample rate in Hz. If None, uses value from + live_options or defaults to the value from StartFrame. + live_options: Deepgram LiveOptions for detailed configuration. If None, + uses sensible defaults (nova-3 model, English, interim results enabled). + **kwargs: Additional arguments passed to the parent STTService. + """ + sample_rate = sample_rate or (live_options.sample_rate if live_options else None) + super().__init__(sample_rate=sample_rate, **kwargs) + + self._endpoint_name = endpoint_name + self._region = region + + # Create default options similar to DeepgramSTTService + default_options = LiveOptions( + encoding="linear16", + language=Language.EN, + model="nova-3", + channels=1, + interim_results=True, + punctuate=True, + ) + + # Merge with provided options + merged_options = default_options.to_dict() + if live_options: + default_model = default_options.model + merged_options.update(live_options.to_dict()) + # Handle the "None" string bug from deepgram-sdk + if "model" in merged_options and merged_options["model"] == "None": + merged_options["model"] = default_model + + # Convert Language enum to string if needed + if "language" in merged_options and isinstance(merged_options["language"], Language): + merged_options["language"] = merged_options["language"].value + + self.set_model_name(merged_options["model"]) + self._settings = merged_options + + self._client: Optional[SageMakerBidiClient] = None + self._response_task: Optional[asyncio.Task] = None + self._keepalive_task: Optional[asyncio.Task] = None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Deepgram SageMaker service supports metrics generation. + """ + return True + + async def set_model(self, model: str): + """Set the Deepgram model and reconnect. + + Disconnects from the current session, updates the model setting, and + establishes a new connection with the updated model. + + Args: + model: The Deepgram model name to use (e.g., "nova-3"). + """ + await super().set_model(model) + logger.info(f"Switching STT model to: [{model}]") + self._settings["model"] = model + await self._disconnect() + await self._connect() + + async def set_language(self, language: Language): + """Set the recognition language and reconnect. + + Disconnects from the current session, updates the language setting, and + establishes a new connection with the updated language. + + Args: + language: The language to use for speech recognition (e.g., Language.EN, + Language.ES). + """ + logger.info(f"Switching STT language to: [{language}]") + self._settings["language"] = language + await self._disconnect() + await self._connect() + + async def start(self, frame: StartFrame): + """Start the Deepgram SageMaker STT service. + + Args: + frame: The start frame containing initialization parameters. + """ + await super().start(frame) + self._settings["sample_rate"] = self.sample_rate + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the Deepgram SageMaker STT service. + + Args: + frame: The end frame. + """ + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the Deepgram SageMaker STT service. + + Args: + frame: The cancel frame. + """ + await super().cancel(frame) + await self._disconnect() + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Send audio data to Deepgram for transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + Frame: None (transcription results come via BiDi stream callbacks). + """ + if self._client and self._client.is_active: + try: + await self._client.send_audio_chunk(audio) + except Exception as e: + logger.error(f"Error sending audio to SageMaker: {e}") + await self.push_error(ErrorFrame(error=f"SageMaker STT error: {e}")) + yield None + + async def _connect(self): + """Connect to the SageMaker endpoint and start the BiDi session. + + Builds the Deepgram query string from settings, creates the BiDi client, + starts the streaming session, and launches background tasks for processing + responses and sending KeepAlive messages. + """ + logger.debug("Connecting to Deepgram on SageMaker...") + + # Update sample rate in settings + self._settings["sample_rate"] = self.sample_rate + + # Build query string from settings, converting booleans to strings + query_params = {} + for key, value in self._settings.items(): + if value is not None: + # Convert boolean values to lowercase strings for Deepgram API + if isinstance(value, bool): + query_params[key] = str(value).lower() + else: + query_params[key] = str(value) + + query_string = "&".join(f"{k}={v}" for k, v in query_params.items()) + + # Create BiDi client + self._client = SageMakerBidiClient( + endpoint_name=self._endpoint_name, + region=self._region, + model_invocation_path="v1/listen", + model_query_string=query_string, + ) + + try: + # Start the session + await self._client.start_session() + + # Start processing responses in the background + self._response_task = self.create_task(self._process_responses()) + + # Start keepalive task to maintain connection + self._keepalive_task = self.create_task(self._send_keepalive()) + + logger.debug("Connected to Deepgram on SageMaker") + await self._call_event_handler("on_connected") + + except Exception as e: + logger.error(f"Failed to connect to SageMaker: {e}") + await self.push_error(ErrorFrame(error=f"SageMaker connection error: {e}")) + await self._call_event_handler("on_connection_error", str(e)) + + async def _disconnect(self): + """Disconnect from the SageMaker endpoint. + + Sends a CloseStream message to Deepgram, cancels background tasks + (KeepAlive and response processing), and closes the BiDi session. + Safe to call multiple times. + """ + if self._client and self._client.is_active: + logger.debug("Disconnecting from Deepgram on SageMaker...") + + # Send CloseStream message to Deepgram + try: + await self._client.send_json({"type": "CloseStream"}) + except Exception as e: + logger.warning(f"Failed to send CloseStream message: {e}") + + # Cancel keepalive task + if self._keepalive_task and not self._keepalive_task.done(): + await self.cancel_task(self._keepalive_task) + + # Cancel response processing task + if self._response_task and not self._response_task.done(): + await self.cancel_task(self._response_task) + + # Close the BiDi session + await self._client.close_session() + + logger.debug("Disconnected from Deepgram on SageMaker") + await self._call_event_handler("on_disconnected") + + async def _send_keepalive(self): + """Send periodic KeepAlive messages to maintain the connection. + + Sends a KeepAlive JSON message to Deepgram every 5 seconds while the + connection is active. This prevents the connection from timing out during + periods of silence. + """ + while self._client and self._client.is_active: + await asyncio.sleep(5) + if self._client and self._client.is_active: + try: + await self._client.send_json({"type": "KeepAlive"}) + except Exception as e: + logger.warning(f"Failed to send KeepAlive: {e}") + + async def _process_responses(self): + """Process streaming responses from Deepgram on SageMaker. + + Continuously receives responses from the BiDi stream, decodes the payload, + parses JSON responses from Deepgram, and processes transcription results. + Runs as a background task until the connection is closed or cancelled. + """ + try: + while self._client and self._client.is_active: + result = await self._client.receive_response() + + if result is None: + break + + # Check if this is a PayloadPart with bytes + if hasattr(result, "value") and hasattr(result.value, "bytes_"): + if result.value.bytes_: + response_data = result.value.bytes_.decode("utf-8") + + try: + # Parse JSON response from Deepgram + parsed = json.loads(response_data) + + # Extract and process transcript if available + if "channel" in parsed: + await self._handle_transcript_response(parsed) + + except json.JSONDecodeError: + logger.warning(f"Non-JSON response: {response_data}") + + except asyncio.CancelledError: + logger.debug("Response processor cancelled") + except Exception as e: + logger.error(f"Error processing responses: {e}", exc_info=True) + await self.push_error(ErrorFrame(error=f"SageMaker response error: {e}")) + finally: + logger.debug("Response processor stopped") + + async def _handle_transcript_response(self, parsed: dict): + """Handle a transcript response from Deepgram. + + Extracts the transcript text, determines if it's final or interim, extracts + language information, and pushes the appropriate frame (TranscriptionFrame + or InterimTranscriptionFrame) downstream. + + Args: + parsed: The parsed JSON response from Deepgram containing channel, + alternatives, transcript, and metadata. + """ + alternatives = parsed.get("channel", {}).get("alternatives", []) + if not alternatives or not alternatives[0].get("transcript"): + return + + transcript = alternatives[0]["transcript"] + if not transcript.strip(): + return + + # Stop TTFB metrics on first transcript + await self.stop_ttfb_metrics() + + is_final = parsed.get("is_final", False) + speech_final = parsed.get("speech_final", False) + + # Extract language if available + language = None + if alternatives[0].get("languages"): + language = alternatives[0]["languages"][0] + language = Language(language) + + if is_final and speech_final: + # Final transcription + await self.push_frame( + TranscriptionFrame( + transcript, + self._user_id, + time_now_iso8601(), + language, + result=parsed, + ) + ) + await self._handle_transcription(transcript, is_final, language) + await self.stop_processing_metrics() + else: + # Interim transcription + await self.push_frame( + InterimTranscriptionFrame( + transcript, + self._user_id, + time_now_iso8601(), + language, + result=parsed, + ) + ) + + @traced_stt + async def _handle_transcription( + self, transcript: str, is_final: bool, language: Optional[Language] = None + ): + """Handle a transcription result with tracing. + + This method is decorated with @traced_stt for observability and tracing + integration. The actual transcription processing is handled by the parent + class and observers. + + Args: + transcript: The transcribed text. + is_final: Whether this is a final transcription result. + language: The detected language of the transcription, if available. + """ + pass + + async def start_metrics(self): + """Start TTFB and processing metrics collection.""" + await self.start_ttfb_metrics() + await self.start_processing_metrics() + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with Deepgram SageMaker-specific handling. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ + await super().process_frame(frame, direction) + + # Start metrics when user starts speaking (if VAD is not provided by Deepgram) + if isinstance(frame, UserStartedSpeakingFrame): + await self.start_metrics() + elif isinstance(frame, UserStoppedSpeakingFrame): + # Send finalize message to Deepgram when user stops speaking + # This tells Deepgram to flush any remaining audio and return final results + if self._client and self._client.is_active: + try: + await self._client.send_json({"type": "Finalize"}) + except Exception as e: + logger.warning(f"Error sending Finalize message: {e}") From 0ece8b5894470997ca17cd4066c4907849afd50d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 22 Nov 2025 07:10:47 -0500 Subject: [PATCH 103/110] Add 07c Deepgram SageMaker example --- CHANGELOG.md | 3 +- env.example | 1 + .../07c-interruptible-deepgram-sagemaker.py | 137 ++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 examples/foundational/07c-interruptible-deepgram-sagemaker.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 93e0ac4dc..aaf7ec85a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added `DeepgramSageMakerSTTService` which connects to a SageMaker hosted - Deepgram STT model. + Deepgram STT model. Added `07c-interruptible-deepgram-sagemaker.py` + foundational example. - Added `SageMakerBidiClient` to connect to SageMaker hosted BiDi compatible services. diff --git a/env.example b/env.example index 2865772ea..33c699259 100644 --- a/env.example +++ b/env.example @@ -44,6 +44,7 @@ DAILY_SAMPLE_ROOM_URL=https://... # Deepgram DEEPGRAM_API_KEY=... +SAGEMAKER_ENDPOINT_NAME=... # DeepSeek DEEPSEEK_API_KEY=... diff --git a/examples/foundational/07c-interruptible-deepgram-sagemaker.py b/examples/foundational/07c-interruptible-deepgram-sagemaker.py new file mode 100644 index 000000000..db230a8ba --- /dev/null +++ b/examples/foundational/07c-interruptible-deepgram-sagemaker.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.aws.llm import AWSBedrockLLMService +from pipecat.services.deepgram.stt_sagemaker import DeepgramSageMakerSTTService +from pipecat.services.deepgram.tts import DeepgramTTSService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + # Initialize Deepgram SageMaker STT Service + # This requires: + # - AWS credentials configured (via environment variables or AWS CLI) + # - A deployed SageMaker endpoint with Deepgram model + stt = DeepgramSageMakerSTTService( + endpoint_name=os.getenv("SAGEMAKER_ENDPOINT_NAME"), + region=os.getenv("AWS_REGION"), + ) + + tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-2-andromeda-en") + + llm = AWSBedrockLLMService( + aws_region=os.getenv("AWS_REGION"), + model="us.amazon.nova-pro-v1:0", + params=AWSBedrockLLMService.InputParams(temperature=0.8), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() From a357ff0205aea7da2987a3a11573037b55f96834 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 22 Nov 2025 07:20:37 -0500 Subject: [PATCH 104/110] Alphabetize the project.optional-dependencies --- pyproject.toml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e4b0a380b..cf83e53ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,9 +54,9 @@ aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.2.0; python_version>='3.12'" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] cerebras = [] -deepseek = [] daily = [ "daily-python~=0.22.0" ] deepgram = [ "deepgram-sdk~=4.7.0" ] +deepseek = [] elevenlabs = [ "pipecat-ai[websockets-base]" ] fal = [ "fal-client~=0.5.9" ] fireworks = [] @@ -69,19 +69,21 @@ gstreamer = [ "pygobject~=3.50.0" ] heygen = [ "livekit>=1.0.13", "pipecat-ai[websockets-base]" ] hume = [ "hume>=0.11.2" ] inworld = [] -krisp = [ "pipecat-ai-krisp~=0.4.0" ] koala = [ "pvkoala~=2.0.3" ] +krisp = [ "pipecat-ai-krisp~=0.4.0" ] langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-openai~=0.3.9" ] livekit = [ "livekit~=1.0.13", "livekit-api~=1.0.5", "tenacity>=8.2.3,<10.0.0" ] lmnt = [ "pipecat-ai[websockets-base]" ] local = [ "pyaudio~=0.2.14" ] +local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "torchaudio>=2.5.0,<3" ] +local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] mcp = [ "mcp[cli]>=1.11.0,<2" ] mem0 = [ "mem0ai~=0.1.94" ] mistral = [] mlx-whisper = [ "mlx-whisper~=0.4.2" ] moondream = [ "accelerate~=1.10.0", "einops~=0.8.0", "pyvips[binary]~=3.0.0", "timm~=1.0.13", "transformers>=4.48.0" ] -nim = [] neuphonic = [ "pipecat-ai[websockets-base]" ] +nim = [] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "pipecat-ai[websockets-base]" ] openpipe = [ "openpipe>=4.50.0,<6" ] @@ -89,16 +91,14 @@ openrouter = [] perplexity = [] playht = [ "pipecat-ai[websockets-base]" ] qwen = [] +remote-smart-turn = [] rime = [ "pipecat-ai[websockets-base]" ] riva = [ "nvidia-riva-client~=2.21.1" ] runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.122.0", "pipecat-ai-small-webrtc-prebuilt>=1.0.0"] +sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"] sambanova = [] sarvam = [ "sarvamai==0.1.21", "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk>=2.28.0,<3" ] -local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "torchaudio>=2.5.0,<3" ] -local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] -remote-smart-turn = [] -sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"] silero = [ "onnxruntime>=1.20.1,<2" ] simli = [ "simli-ai~=1.0.3"] soniox = [ "pipecat-ai[websockets-base]" ] From 12c29b71f312a9abb3c0f2ece5e23b246362565f Mon Sep 17 00:00:00 2001 From: fbarril Date: Mon, 24 Nov 2025 23:27:13 +0000 Subject: [PATCH 105/110] add entry to CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 843fd98c4..2df70ebc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `LiveKitRESTHelper` utility class for managing LiveKit rooms via REST API. + - Added optional speaking rate control to `InworldTTSService`. ### Changed From 60da46637920b3e7d5eb637a0ef0fbea45475602 Mon Sep 17 00:00:00 2001 From: fbarril Date: Mon, 24 Nov 2025 23:27:32 +0000 Subject: [PATCH 106/110] add pyjwt as a livekit dependency --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 149bf18a4..ff1452f11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ inworld = [] krisp = [ "pipecat-ai-krisp~=0.4.0" ] koala = [ "pvkoala~=2.0.3" ] langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-openai~=0.3.9" ] -livekit = [ "livekit~=1.0.13", "livekit-api~=1.0.5", "tenacity>=8.2.3,<10.0.0" ] +livekit = [ "livekit~=1.0.13", "livekit-api~=1.0.5", "tenacity>=8.2.3,<10.0.0", "pyjwt>=2.10.1" ] lmnt = [ "pipecat-ai[websockets-base]" ] local = [ "pyaudio~=0.2.14" ] mcp = [ "mcp[cli]>=1.11.0,<2" ] @@ -112,7 +112,6 @@ webrtc = [ "aiortc>=1.13.0,<2", "opencv-python>=4.11.0.86,<5" ] websocket = [ "pipecat-ai[websockets-base]", "fastapi>=0.115.6,<0.122.0" ] websockets-base = [ "websockets>=13.1,<16.0" ] whisper = [ "faster-whisper~=1.1.1" ] -auth = [ "pyjwt>=2.10.1" ] [dependency-groups] dev = [ From e2161ea63d6b77bbad6405124282336b71489e9f Mon Sep 17 00:00:00 2001 From: fbarril Date: Mon, 24 Nov 2025 23:30:11 +0000 Subject: [PATCH 107/110] add pyjwt as a livekit dependency --- uv.lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index eb2fca39c..37177a822 100644 --- a/uv.lock +++ b/uv.lock @@ -4522,6 +4522,7 @@ langchain = [ livekit = [ { name = "livekit" }, { name = "livekit-api" }, + { name = "pyjwt" }, { name = "tenacity" }, ] lmnt = [ @@ -4739,6 +4740,7 @@ requires-dist = [ { name = "pyaudio", marker = "extra == 'local'", specifier = "~=0.2.14" }, { name = "pydantic", specifier = ">=2.10.6,<3" }, { name = "pygobject", marker = "extra == 'gstreamer'", specifier = "~=3.50.0" }, + { name = "pyjwt", marker = "extra == 'livekit'", specifier = ">=2.10.1" }, { name = "pyloudnorm", specifier = "~=0.1.1" }, { name = "python-dotenv", marker = "extra == 'runner'", specifier = ">=1.0.0,<2.0.0" }, { name = "pyvips", extras = ["binary"], marker = "extra == 'moondream'", specifier = "~=3.0.0" }, @@ -4763,7 +4765,7 @@ requires-dist = [ { name = "wait-for2", marker = "python_full_version < '3.12'", specifier = ">=0.4.1" }, { name = "websockets", marker = "extra == 'websockets-base'", specifier = ">=13.1,<16.0" }, ] -provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "cerebras", "deepseek", "daily", "deepgram", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "krisp", "koala", "langchain", "livekit", "lmnt", "local", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "nim", "neuphonic", "noisereduce", "openai", "openpipe", "openrouter", "perplexity", "playht", "qwen", "rime", "riva", "runner", "sambanova", "sarvam", "sentry", "local-smart-turn", "local-smart-turn-v3", "remote-smart-turn", "sagemaker", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] +provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "cerebras", "daily", "deepgram", "deepseek", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "koala", "krisp", "langchain", "livekit", "lmnt", "local", "local-smart-turn", "local-smart-turn-v3", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "neuphonic", "nim", "noisereduce", "openai", "openpipe", "openrouter", "perplexity", "playht", "qwen", "remote-smart-turn", "rime", "riva", "runner", "sagemaker", "sambanova", "sarvam", "sentry", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] [package.metadata.requires-dev] dev = [ From 9e4ec4f7f332a9814a79e5bd535628545eaa3743 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 21 Nov 2025 00:32:14 -0500 Subject: [PATCH 108/110] Implement `AWSBedrockAgentCoreProcessor` --- pyproject.toml | 2 +- src/pipecat/services/aws/__init__.py | 1 + src/pipecat/services/aws/agent_core.py | 266 +++++++++++++++++++++++++ uv.lock | 32 +-- 4 files changed, 284 insertions(+), 17 deletions(-) create mode 100644 src/pipecat/services/aws/agent_core.py diff --git a/pyproject.toml b/pyproject.toml index cf83e53ee..f5ade679b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ aic = [ "aic-sdk~=1.1.0" ] anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] -aws = [ "aioboto3~=15.0.0", "pipecat-ai[websockets-base]" ] +aws = [ "aioboto3~=15.5.0", "pipecat-ai[websockets-base]" ] aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.2.0; python_version>='3.12'" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] diff --git a/src/pipecat/services/aws/__init__.py b/src/pipecat/services/aws/__init__.py index 6f6903f75..88725f965 100644 --- a/src/pipecat/services/aws/__init__.py +++ b/src/pipecat/services/aws/__init__.py @@ -8,6 +8,7 @@ import sys from pipecat.services import DeprecatedModuleProxy +from .agent_core import * from .llm import * from .nova_sonic import * from .sagemaker import * diff --git a/src/pipecat/services/aws/agent_core.py b/src/pipecat/services/aws/agent_core.py new file mode 100644 index 000000000..8295062bf --- /dev/null +++ b/src/pipecat/services/aws/agent_core.py @@ -0,0 +1,266 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""AWS AgentCore Processor Module. + +This module defines the AWSAgentCoreProcessor, which invokes agents hosted on +Amazon Bedrock AgentCore Runtime and streams their responses as LLMTextFrames. +""" + +import asyncio +import json +import os +from typing import Callable, Optional + +import aioboto3 +from loguru import logger + +from pipecat.frames.frames import ( + Frame, + LLMContextFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, +) +from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + + +def default_context_to_payload_transformer( + context: LLMContext | OpenAILLMContext, +) -> Optional[str]: + """Default transformer to create AgentCore payload from LLM context. + + Extracts the latest user or system message text and wraps it in {"prompt": ""}. + + Args: + context: The LLM context containing conversation messages. + + Returns: + A JSON string payload for AgentCore, or None if no valid message found. + """ + messages = context.messages + + if not messages: + return None + + last_message = messages[-1] + if isinstance(last_message, LLMSpecificMessage) or last_message.get("role") not in ( + "user", + "system", + ): + return None + + content = last_message.get("content") + if not content: + return None + + if isinstance(content, str): + prompt = content + elif isinstance(content, list): + prompt = " ".join([part.get("text", "") for part in content]) + else: + return None + + return json.dumps({"prompt": prompt}) + + +def default_response_to_output_transformer(response_line: str) -> Optional[str]: + """Default transformer to extract output text from AgentCore response. + + Expects responses with {"response": ""} format. + + Args: + response_line: The raw response line from AgentCore (without "data: " prefix). + + Returns: + The extracted output text, or None if no text found. + """ + response_json = json.loads(response_line) + return response_json.get("response") + + +class AWSAgentCoreProcessor(FrameProcessor): + """Processor that runs an Amazon Bedrock AgentCore agent. + + Input: + - LLMContextFrame: Supplies a context used to invoke the agent. + + Output: + - LLMTextFrame: The agent's text response(s). + A single agent invocation may result in multiple text frames. + + This processor transforms the input context to a payload for the AgentCore + agent, and transforms the agent's response(s) into output text frame(s). Both + mappings are configurable via transformers. Below is the default behavior. + + Input transformer (context_to_payload_transformer): + - Grabs the latest user or system message (if it's the latest message) + - Extracts its text content + - Constructs a payload that looks like {"prompt": ""} + + Output transformer (response_to_output_transformer): + - Expects responses that look like {"response": ""} + - Extracts the text for use in the LLMTextFrame(s) + """ + + def __init__( + self, + agentArn: str, + aws_access_key: Optional[str] = None, + aws_secret_key: Optional[str] = None, + aws_session_token: Optional[str] = None, + aws_region: Optional[str] = None, + context_to_payload_transformer: Optional[ + Callable[[LLMContext | OpenAILLMContext], Optional[str]] + ] = None, + response_to_output_transformer: Optional[Callable[[str], Optional[str]]] = None, + **kwargs, + ): + """Initialize the AWS AgentCore processor. + + Args: + agentArn: The Amazon Web Services Resource Name (ARN) of the agent. + aws_access_key: AWS access key ID. If None, uses default credentials. + aws_secret_key: AWS secret access key. If None, uses default credentials. + aws_session_token: AWS session token for temporary credentials. + aws_region: AWS region. + context_to_payload_transformer: Optional callable to transform + LLMContext into AgentCore payload string. If None, uses + default_context_to_payload_transformer. + response_to_output_transformer: Optional callable to extract output text + from AgentCore response. If None, uses + default_response_to_output_transformer. + **kwargs: Additional arguments passed to parent FrameProcessor. + """ + super().__init__(**kwargs) + + self._agentArn = agentArn + self._aws_session = aioboto3.Session() + + # Store AWS session parameters for creating client in async context + self._aws_params = { + "aws_access_key_id": aws_access_key or os.getenv("AWS_ACCESS_KEY_ID"), + "aws_secret_access_key": aws_secret_key or os.getenv("AWS_SECRET_ACCESS_KEY"), + "aws_session_token": aws_session_token or os.getenv("AWS_SESSION_TOKEN"), + "region_name": aws_region or os.getenv("AWS_REGION", "us-east-1"), + } + + # Set transformers with defaults + self._context_to_payload_transformer = ( + context_to_payload_transformer or default_context_to_payload_transformer + ) + self._response_to_output_transformer = ( + response_to_output_transformer or default_response_to_output_transformer + ) + + # State for managing output response bookends + self._output_response_open = False + self._last_text_frame_time: Optional[float] = None + self._close_task: Optional[asyncio.Task] = None + self._output_response_timeout = 1.0 # seconds + + async def _close_output_response_after_timeout(self): + """Close the output response after timeout if no new text frames arrive.""" + await asyncio.sleep(self._output_response_timeout) + if self._output_response_open: + self._output_response_open = False + await self.push_frame(LLMFullResponseEndFrame()) + + async def _push_text_frame(self, text: str): + """Push a text frame, managing output response bookends.""" + # Cancel any pending close task + if self._close_task and not self._close_task.done(): + self._close_task.cancel() + try: + await self._close_task + except asyncio.CancelledError: + pass + + # Open output response if needed + if not self._output_response_open: + await self.push_frame(LLMFullResponseStartFrame()) + self._output_response_open = True + + # Push the text frame + await self.push_frame(LLMTextFrame(text)) + self._last_text_frame_time = asyncio.get_event_loop().time() + + # Schedule closing the output response after timeout + self._close_task = asyncio.create_task(self._close_output_response_after_timeout()) + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle LLM message frames. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + if isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): + # Create payload to invoke AgentCore agent + payload = self._context_to_payload_transformer(frame.context) + + if not payload: + return + + async with self._aws_session.client("bedrock-agentcore", **self._aws_params) as client: + # Invoke the AgentCore agent + response = await client.invoke_agent_runtime( + agentRuntimeArn=self._agentArn, payload=payload.encode() + ) + + # Determine if this is a streamed multi-part response, which + # will affect our parsing + is_multi_part_response = "text/event-stream" in response.get("contentType", "") + + # Handle each response part (there may be one, for single + # responses, or multiple, for streamed multi-part responses) + async for part in response.get("response", []): + part_string = part.decode("utf-8") + + # In streamed multi-part responses, each part might have + # one or more lines, each of which starts with "data: ". + # Treat each line as a response. + if is_multi_part_response: + for line in part_string.split("\n"): + # Get response text from this line + if not line: + continue + if not line.startswith("data: "): + logger.warning(f"Expected line to start with 'data: ', got: {line}") + continue + line = line[6:] # omit "data: " + + # Transform response line to output text + text = self._response_to_output_transformer(line) + if text: + await self._push_text_frame(text) + + # In single-part responses, the whole part is one response + # and there's no "data: " prefix + else: + # Transform response part string to output text + text = self._response_to_output_transformer(part_string) + if text: + await self._push_text_frame(text) + + # Final close if output response is still open after all parts processed + if self._output_response_open: + if self._close_task and not self._close_task.done(): + self._close_task.cancel() + try: + await self._close_task + except asyncio.CancelledError: + pass + self._output_response_open = False + await self.push_frame(LLMFullResponseEndFrame()) + else: + await self.push_frame(frame, direction) diff --git a/uv.lock b/uv.lock index eb2fca39c..7a937e54d 100644 --- a/uv.lock +++ b/uv.lock @@ -45,20 +45,20 @@ sdist = { url = "https://files.pythonhosted.org/packages/99/83/bf38b95d98c67b8eb [[package]] name = "aioboto3" -version = "15.0.0" +version = "15.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiobotocore", extra = ["boto3"] }, { name = "aiofiles" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/80/d0/ed107e16551ba1b93ddcca9a6bf79580450945268a8bc396530687b3189f/aioboto3-15.0.0.tar.gz", hash = "sha256:dce40b701d1f8e0886dc874d27cd9799b8bf6b32d63743f57e7bef7e4a562756", size = 225278, upload-time = "2025-06-26T16:30:48.967Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/01/92e9ab00f36e2899315f49eefcd5b4685fbb19016c7f19a9edf06da80bb0/aioboto3-15.5.0.tar.gz", hash = "sha256:ea8d8787d315594842fbfcf2c4dce3bac2ad61be275bc8584b2ce9a3402a6979", size = 255069, upload-time = "2025-10-30T13:37:16.122Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/95/d69c744f408e5e4592fe53ed98fc244dd13b83d84cf1f83b2499d98bfcc9/aioboto3-15.0.0-py3-none-any.whl", hash = "sha256:9cf54b3627c8b34bb82eaf43ab327e7027e37f92b1e10dd5cfe343cd512568d0", size = 35785, upload-time = "2025-06-26T16:30:47.444Z" }, + { url = "https://files.pythonhosted.org/packages/e5/3e/e8f5b665bca646d43b916763c901e00a07e40f7746c9128bdc912a089424/aioboto3-15.5.0-py3-none-any.whl", hash = "sha256:cc880c4d6a8481dd7e05da89f41c384dbd841454fc1998ae25ca9c39201437a6", size = 35913, upload-time = "2025-10-30T13:37:14.549Z" }, ] [[package]] name = "aiobotocore" -version = "2.23.0" +version = "2.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -69,9 +69,9 @@ dependencies = [ { name = "python-dateutil" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/25/4b06ea1214ddf020a28df27dc7136ac9dfaf87929d51e6f6044dd350ed67/aiobotocore-2.23.0.tar.gz", hash = "sha256:0333931365a6c7053aee292fe6ef50c74690c4ae06bb019afdf706cb6f2f5e32", size = 115825, upload-time = "2025-06-12T23:46:38.055Z" } +sdist = { url = "https://files.pythonhosted.org/packages/62/94/2e4ec48cf1abb89971cb2612d86f979a6240520f0a659b53a43116d344dc/aiobotocore-2.25.1.tar.gz", hash = "sha256:ea9be739bfd7ece8864f072ec99bb9ed5c7e78ebb2b0b15f29781fbe02daedbc", size = 120560, upload-time = "2025-10-28T22:33:21.787Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/43/ccf9b29669cdb09fd4bfc0a8effeb2973b22a0f3c3be4142d0b485975d11/aiobotocore-2.23.0-py3-none-any.whl", hash = "sha256:8202cebbf147804a083a02bc282fbfda873bfdd0065fd34b64784acb7757b66e", size = 84161, upload-time = "2025-06-12T23:46:36.305Z" }, + { url = "https://files.pythonhosted.org/packages/95/2a/d275ec4ce5cd0096665043995a7d76f5d0524853c76a3d04656de49f8808/aiobotocore-2.25.1-py3-none-any.whl", hash = "sha256:eb6daebe3cbef5b39a0bb2a97cffbe9c7cb46b2fcc399ad141f369f3c2134b1f", size = 86039, upload-time = "2025-10-28T22:33:19.949Z" }, ] [package.optional-dependencies] @@ -620,30 +620,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.38.27" +version = "1.40.61" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/96/fc74d8521d2369dd8c412438401ff12e1350a1cd3eab5c758ed3dd5e5f82/boto3-1.38.27.tar.gz", hash = "sha256:94bd7fdd92d5701b362d4df100d21e28f8307a67ff56b6a8b0398119cf22f859", size = 111875, upload-time = "2025-05-30T19:32:41.352Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/f9/6ef8feb52c3cce5ec3967a535a6114b57ac7949fd166b0f3090c2b06e4e5/boto3-1.40.61.tar.gz", hash = "sha256:d6c56277251adf6c2bdd25249feae625abe4966831676689ff23b4694dea5b12", size = 111535, upload-time = "2025-10-28T19:26:57.247Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/43/8b/b2361188bd1e293eede1bc165e2461d390394f71ec0c8c21211c8dabf62c/boto3-1.38.27-py3-none-any.whl", hash = "sha256:95f5fe688795303a8a15e8b7e7f255cadab35eae459d00cc281a4fd77252ea80", size = 139938, upload-time = "2025-05-30T19:32:38.006Z" }, + { url = "https://files.pythonhosted.org/packages/61/24/3bf865b07d15fea85b63504856e137029b6acbc73762496064219cdb265d/boto3-1.40.61-py3-none-any.whl", hash = "sha256:6b9c57b2a922b5d8c17766e29ed792586a818098efe84def27c8f582b33f898c", size = 139321, upload-time = "2025-10-28T19:26:55.007Z" }, ] [[package]] name = "botocore" -version = "1.38.27" +version = "1.40.61" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/36/5e/67899214ad57f7f26af5bd776ac5eb583dc4ecf5c1e52e2cbfdc200e487a/botocore-1.38.27.tar.gz", hash = "sha256:9788f7efe974328a38cbade64cc0b1e67d27944b899f88cb786ae362973133b6", size = 13919963, upload-time = "2025-05-30T19:32:29.657Z" } +sdist = { url = "https://files.pythonhosted.org/packages/28/a3/81d3a47c2dbfd76f185d3b894f2ad01a75096c006a2dd91f237dca182188/botocore-1.40.61.tar.gz", hash = "sha256:a2487ad69b090f9cccd64cf07c7021cd80ee9c0655ad974f87045b02f3ef52cd", size = 14393956, upload-time = "2025-10-28T19:26:46.108Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/83/a753562020b69fa90cebc39e8af2c753b24dcdc74bee8355ee3f6cefdf34/botocore-1.38.27-py3-none-any.whl", hash = "sha256:a785d5e9a5eda88ad6ab9ed8b87d1f2ac409d0226bba6ff801c55359e94d91a8", size = 13580545, upload-time = "2025-05-30T19:32:26.712Z" }, + { url = "https://files.pythonhosted.org/packages/38/c5/f6ce561004db45f0b847c2cd9b19c67c6bf348a82018a48cb718be6b58b0/botocore-1.40.61-py3-none-any.whl", hash = "sha256:17ebae412692fd4824f99cde0f08d50126dc97954008e5ba2b522eb049238aa7", size = 14055973, upload-time = "2025-10-28T19:26:42.15Z" }, ] [[package]] @@ -4665,7 +4665,7 @@ docs = [ requires-dist = [ { name = "accelerate", marker = "extra == 'moondream'", specifier = "~=1.10.0" }, { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.1.0" }, - { name = "aioboto3", marker = "extra == 'aws'", specifier = "~=15.0.0" }, + { name = "aioboto3", marker = "extra == 'aws'", specifier = "~=15.5.0" }, { name = "aiofiles", specifier = ">=24.1.0,<25" }, { name = "aiohttp", specifier = ">=3.11.12,<4" }, { name = "aiortc", marker = "extra == 'webrtc'", specifier = ">=1.13.0,<2" }, @@ -6220,14 +6220,14 @@ wheels = [ [[package]] name = "s3transfer" -version = "0.13.1" +version = "0.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6d/05/d52bf1e65044b4e5e27d4e63e8d1579dbdec54fce685908ae09bc3720030/s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf", size = 150589, upload-time = "2025-07-18T19:22:42.31Z" } +sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547, upload-time = "2025-09-09T19:23:31.089Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/4f/d073e09df851cfa251ef7840007d04db3293a0482ce607d2b993926089be/s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724", size = 85308, upload-time = "2025-07-18T19:22:40.947Z" }, + { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, ] [[package]] From 5907b51c7d4be2ffdf26dbc67953eca96b626e66 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 24 Nov 2025 17:24:28 -0500 Subject: [PATCH 109/110] In `AWSBedrockAgentCoreProcessor` use `self.create_task()`/`self.cancel_task()` instead of using `asyncio` directly. --- src/pipecat/services/aws/agent_core.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/pipecat/services/aws/agent_core.py b/src/pipecat/services/aws/agent_core.py index 8295062bf..be4806221 100644 --- a/src/pipecat/services/aws/agent_core.py +++ b/src/pipecat/services/aws/agent_core.py @@ -178,11 +178,7 @@ class AWSAgentCoreProcessor(FrameProcessor): """Push a text frame, managing output response bookends.""" # Cancel any pending close task if self._close_task and not self._close_task.done(): - self._close_task.cancel() - try: - await self._close_task - except asyncio.CancelledError: - pass + await self.cancel_task(self._close_task) # Open output response if needed if not self._output_response_open: @@ -194,7 +190,7 @@ class AWSAgentCoreProcessor(FrameProcessor): self._last_text_frame_time = asyncio.get_event_loop().time() # Schedule closing the output response after timeout - self._close_task = asyncio.create_task(self._close_output_response_after_timeout()) + self._close_task = self.create_task(self._close_output_response_after_timeout()) async def process_frame(self, frame: Frame, direction: FrameDirection): """Process incoming frames and handle LLM message frames. @@ -255,11 +251,7 @@ class AWSAgentCoreProcessor(FrameProcessor): # Final close if output response is still open after all parts processed if self._output_response_open: if self._close_task and not self._close_task.done(): - self._close_task.cancel() - try: - await self._close_task - except asyncio.CancelledError: - pass + await self.cancel_task(self._close_task) self._output_response_open = False await self.push_frame(LLMFullResponseEndFrame()) else: From fa0100c38bd1da7ab93170f71177a4d826ef659a Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 24 Nov 2025 20:04:18 -0500 Subject: [PATCH 110/110] fix: remove stt_sagemaker import from deepgram/__init__.py --- src/pipecat/services/deepgram/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipecat/services/deepgram/__init__.py b/src/pipecat/services/deepgram/__init__.py index 227ac5c64..c23ebbec9 100644 --- a/src/pipecat/services/deepgram/__init__.py +++ b/src/pipecat/services/deepgram/__init__.py @@ -10,7 +10,6 @@ from pipecat.services import DeprecatedModuleProxy from .flux import * from .stt import * -from .stt_sagemaker import * from .tts import * sys.modules[__name__] = DeprecatedModuleProxy(globals(), "deepgram", "deepgram.[stt,tts]")