From 9bf88bbf14eb15e50e84b8603b57eebbeeaadb14 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 07:43:30 -0300 Subject: [PATCH 01/13] Fixing RivaTTSService error handler. --- src/pipecat/services/riva/tts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py index 07aa4d105..4d393943b 100644 --- a/src/pipecat/services/riva/tts.py +++ b/src/pipecat/services/riva/tts.py @@ -166,7 +166,6 @@ class RivaTTSService(TTSService): add_response(None) except Exception as e: logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") add_response(None) await self.start_ttfb_metrics() @@ -191,6 +190,7 @@ class RivaTTSService(TTSService): resp = await asyncio.wait_for(queue.get(), timeout=RIVA_TTS_TIMEOUT_SECS) except asyncio.TimeoutError: logger.error(f"{self} timeout waiting for audio response") + yield ErrorFrame(error=f"{self} error: {e}") await self.start_tts_usage_metrics(text) yield TTSStoppedFrame() From 77cd106795778fa22bcee3ba64588e7e79b9529a Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 09:54:08 -0300 Subject: [PATCH 02/13] Extracted the logic for retrying connections, and create a new send_with_retry method inside WebSocketService. --- src/pipecat/services/websocket_service.py | 74 +++++++++++++++++------ 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index f9799b9c3..d19e6ad4d 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -36,6 +36,7 @@ class WebsocketService(ABC): """ self._websocket: Optional[websockets.WebSocketClientProtocol] = None self._reconnect_on_error = reconnect_on_error + self._reconnect_in_progress: bool = False # Add this flag async def _verify_connection(self) -> bool: """Verify the websocket connection is active and responsive. @@ -66,6 +67,59 @@ class WebsocketService(ABC): await self._connect_websocket() return await self._verify_connection() + async def _try_reconnect( + self, + max_retries: int = 3, + report_error: Optional[Callable[[ErrorFrame], Awaitable[None]]] = None, + ) -> bool: + # Prevent concurrent reconnection attempts + if self._reconnect_in_progress: + logger.warning(f"{self} reconnect attempt aborted: already in progress") + return False + + self._reconnect_in_progress = True + last_exception: Optional[Exception] = None + try: + for attempt in range(1, max_retries + 1): + try: + logger.warning(f"{self} reconnecting, attempt {attempt}") + if await self._reconnect_websocket(attempt): + logger.info(f"{self} reconnected successfully on attempt {attempt}") + return True + except Exception as e: + last_exception = e + logger.error(f"{self} reconnection attempt {attempt} failed: {e}") + if report_error: + await report_error( + ErrorFrame(f"{self} reconnection attempt {attempt} failed: {e}") + ) + wait_time = exponential_backoff_time(attempt) + await asyncio.sleep(wait_time) + fatal_msg = f"{self} failed to reconnect after {max_retries} attempts" + if last_exception: + fatal_msg += f": {last_exception}" + logger.error(fatal_msg) + if report_error: + await report_error(ErrorFrame(fatal_msg, fatal=True)) + return False + finally: + self._reconnect_in_progress = False + + async def send_with_retry(self, message, report_error: Callable[[ErrorFrame], Awaitable[None]]): + """Attempt to send a message, retrying after reconnect if necessary.""" + try: + await self._websocket.send(message) + except Exception as e: + logger.error(f"{self} send failed: {e}, will try to reconnect") + # Try to reconnect before retrying + success = await self._try_reconnect(report_error=report_error) + if success: + logger.info(f"{self} reconnected successfully, will retry send the message") + # trying to send the message one more time + await self._websocket.send(message) + else: + logger.error(f"{self} send failed; unable to reconnect") + async def _receive_task_handler(self, report_error: Callable[[ErrorFrame], Awaitable[None]]): """Handle websocket message receiving with automatic retry logic. @@ -76,13 +130,9 @@ class WebsocketService(ABC): Args: report_error: Callback function to report connection errors. """ - retry_count = 0 - MAX_RETRIES = 3 - while True: try: await self._receive_messages() - retry_count = 0 # Reset counter on successful message receive except ConnectionClosedOK as e: # Normal closure, don't retry logger.debug(f"{self} connection closed normally: {e}") @@ -92,21 +142,9 @@ class WebsocketService(ABC): logger.error(message) if self._reconnect_on_error: - retry_count += 1 - if retry_count >= MAX_RETRIES: - await report_error(ErrorFrame(message)) + success = await self._try_reconnect(report_error=report_error) + if not success: break - - logger.warning(f"{self} connection error, will retry: {e}") - await report_error(ErrorFrame(message)) - - try: - if await self._reconnect_websocket(retry_count): - retry_count = 0 # Reset counter on successful reconnection - wait_time = exponential_backoff_time(retry_count) - await asyncio.sleep(wait_time) - except Exception as reconnect_error: - logger.error(f"{self} reconnection failed: {reconnect_error}") else: await report_error(ErrorFrame(message)) break From 19cc0177b85bea279692abb1d829567667d73027 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 09:54:20 -0300 Subject: [PATCH 03/13] Refactored DeepgramFluxSTTService to automatically reconnect if sending a message fails. --- src/pipecat/services/deepgram/flux/stt.py | 114 ++++++++++++++++++---- 1 file changed, 95 insertions(+), 19 deletions(-) diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index ac6e8bb08..a0d45afe4 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -6,7 +6,9 @@ """Deepgram Flux speech-to-text service implementation.""" +import asyncio import json +import time from enum import Enum from typing import Any, AsyncGenerator, Dict, Optional from urllib.parse import urlencode @@ -94,6 +96,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): mip_opt_out: Optional. Opts out requests from the Deepgram Model Improvement Program (default False). tag: List of tags to label requests for identification during usage reporting. + min_confidence: Optional. Minimum confidence required confidence to create a TranscriptionFrame """ eager_eot_threshold: Optional[float] = None @@ -102,6 +105,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): keyterm: list = [] mip_opt_out: Optional[bool] = None tag: list = [] + min_confidence: Optional[float] = None # New parameter def __init__( self, @@ -163,6 +167,13 @@ class DeepgramFluxSTTService(WebsocketSTTService): self._register_event_handler("on_end_of_turn") self._register_event_handler("on_eager_end_of_turn") self._register_event_handler("on_update") + self._connection_established_event = asyncio.Event() + # Watchdog task to prevent dangling tasks + # If we stop sending audio to Flux after we have received that the User has started speaking + # we never receive the user stopped speaking event unless we resume sending audio to it. + self._last_stt_time = None + self._watchdog_task = None + self._user_is_speaking = False async def _connect(self): """Connect to WebSocket and start background tasks. @@ -172,9 +183,6 @@ class DeepgramFluxSTTService(WebsocketSTTService): """ await self._connect_websocket() - if self._websocket and not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) - async def _disconnect(self): """Disconnect from WebSocket and clean up tasks. @@ -182,14 +190,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): and cleans up resources to prevent memory leaks. """ try: - # Cancel background tasks BEFORE closing websocket - if self._receive_task: - await self.cancel_task(self._receive_task, timeout=2.0) - self._receive_task = None - - # Now close the websocket await self._disconnect_websocket() - except Exception as e: logger.error(f"{self} exception: {e}") await self.push_error(ErrorFrame(error=f"{self} error: {e}")) @@ -197,6 +198,25 @@ class DeepgramFluxSTTService(WebsocketSTTService): # Reset state only after everything is cleaned up self._websocket = None + async def _send_silence(self, duration_secs: float = 0.5): + """Send a block of silence of the specified duration (default 500 ms).""" + sample_width = 2 # bytes per sample for 16-bit PCM + num_channels = 1 # mono + num_samples = int(self.sample_rate * duration_secs) + silence = b"\x00" * (num_samples * sample_width * num_channels) + await self._websocket.send(silence) + + async def _watchdog_task_handler(self): + while self._websocket and self._websocket.state is State.OPEN: + now = time.monotonic() + # More than 500 ms without sending new audio to Flux + if self._user_is_speaking and self._last_stt_time and now - self._last_stt_time > 0.5: + logger.warning("Sending silence to Flux to prevent dangling task") + await self._send_silence() + self._last_stt_time = time.monotonic() + # check every 100ms + await asyncio.sleep(0.1) + async def _connect_websocket(self): """Establish WebSocket connection to API. @@ -208,10 +228,26 @@ class DeepgramFluxSTTService(WebsocketSTTService): if self._websocket and self._websocket.state is State.OPEN: return + self._connection_established_event.clear() + self._user_is_speaking = False self._websocket = await websocket_connect( self._websocket_url, additional_headers={"Authorization": f"Token {self._api_key}"}, ) + + # Creating the receiver task + if not self._receive_task: + self._receive_task = self.create_task( + self._receive_task_handler(self._report_error) + ) + + # Creating the watchdog task + if not self._watchdog_task: + self._watchdog_task = self.create_task(self._watchdog_task_handler()) + + # Now wait for the connection established event + logger.debug("WebSocket connected, waiting for server confirmation...") + await self._connection_established_event.wait() logger.debug("Connected to Deepgram Flux Websocket") await self._call_event_handler("on_connected") except Exception as e: @@ -227,6 +263,16 @@ class DeepgramFluxSTTService(WebsocketSTTService): metrics collection. Handles disconnection errors gracefully. """ try: + # Cancel background tasks BEFORE closing websocket + if self._receive_task: + await self.cancel_task(self._receive_task, timeout=2.0) + self._receive_task = None + if self._watchdog_task: + await self.cancel_task(self._watchdog_task, timeout=2.0) + self._watchdog_task = None + self._last_stt_time = None + + self._connection_established_event.clear() await self.stop_all_metrics() if self._websocket: @@ -340,7 +386,8 @@ class DeepgramFluxSTTService(WebsocketSTTService): return try: - await self._websocket.send(audio) + self._last_stt_time = time.monotonic() + await self.send_with_retry(audio, self._report_error) except Exception as e: logger.error(f"{self} exception: {e}") yield ErrorFrame(error=f"{self} error: {e}") @@ -463,6 +510,8 @@ class DeepgramFluxSTTService(WebsocketSTTService): transcription processing. """ logger.info("Connected to Flux - ready to stream audio") + # Notify connection is established + self._connection_established_event.set() async def _handle_fatal_error(self, data: Dict[str, Any]): """Handle fatal error messages from Deepgram Flux. @@ -530,6 +579,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): transcript: maybe the first few words of the turn. """ logger.debug("User started speaking") + self._user_is_speaking = True await self.push_interruption_task_frame_and_wait() await self.broadcast_frame(UserStartedSpeakingFrame) await self.start_metrics() @@ -550,6 +600,22 @@ class DeepgramFluxSTTService(WebsocketSTTService): logger.trace(f"Received event TurnResumed: {event}") await self._call_event_handler("on_turn_resumed") + def _calculate_average_confidence(self, transcript_data) -> Optional[float]: + """Calculate the average confidence from transcript data. + + Return None if the data is missing or invalid. + """ + # Example: Assume transcript_data has a list of words with confidence + words = transcript_data.get("words") + if not words or not isinstance(words, list): + return None + confidences = [ + w.get("confidence") for w in words if isinstance(w.get("confidence"), (float, int)) + ] + if not confidences: + return None + return sum(confidences) / len(confidences) + async def _handle_end_of_turn(self, transcript: str, data: Dict[str, Any]): """Handle EndOfTurn events from Deepgram Flux. @@ -569,16 +635,26 @@ class DeepgramFluxSTTService(WebsocketSTTService): data: The TurnInfo message data containing event type, transcript and some extra metadata. """ logger.debug("User stopped speaking") + self._user_is_speaking = False - await self.push_frame( - TranscriptionFrame( - transcript, - self._user_id, - time_now_iso8601(), - self._language, - result=data, + # Compute the average confidence + average_confidence = self._calculate_average_confidence(data) + + if not self._params.min_confidence or average_confidence > self._params.min_confidence: + await self.push_frame( + TranscriptionFrame( + transcript, + self._user_id, + time_now_iso8601(), + self._language, + result=data, + ) ) - ) + else: + logger.warning( + f"Transcription confidence below min_confidence threshold: {average_confidence}" + ) + await self._handle_transcription(transcript, True, self._language) await self.stop_processing_metrics() await self.push_frame(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM) From 04dbbabc036289fb3d90e5228a3e5af573d1ba21 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 09:54:30 -0300 Subject: [PATCH 04/13] Introduced a minimum confidence parameter in DeepgramFluxSTTService to avoid generating transcriptions below a defined threshold. --- examples/foundational/07c-interruptible-deepgram-flux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/foundational/07c-interruptible-deepgram-flux.py b/examples/foundational/07c-interruptible-deepgram-flux.py index f5b246acd..62579c2c5 100644 --- a/examples/foundational/07c-interruptible-deepgram-flux.py +++ b/examples/foundational/07c-interruptible-deepgram-flux.py @@ -52,7 +52,10 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramFluxSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramFluxSTTService( + api_key=os.getenv("DEEPGRAM_API_KEY"), + params=DeepgramFluxSTTService.InputParams(min_confidence=0.3), + ) tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-2-andromeda-en") From b104a59b109168c01c0240ae885f7b3a0fd7e2ce Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 09:54:39 -0300 Subject: [PATCH 05/13] Mentioning the Deepgram Flux improvements in the changelog. --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cda253c7..90a7542de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a watchdog to `DeepgramFluxSTTService` to prevent dangling tasks in case the + user was speaking and we stop receiving audio. + +- Introduced a minimum confidence parameter in `DeepgramFluxSTTService` to avoid + generating transcriptions below a defined threshold. + - Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT service from ElevenLabs. @@ -18,6 +24,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Extracted the logic for retrying connections, and create a new `send_with_retry` + method inside `WebSocketService`. + +- Refactored `DeepgramFluxSTTService` to automatically reconnect if sending a + message fails. + - Updated all STT and TTS services to use consistent error handling pattern with `push_error()` method for better pipeline error event integration. From f3b254e335be513623a6b30b80e1c60353d1f22e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 14 Nov 2025 21:39:18 -0500 Subject: [PATCH 06/13] D'oh! My TTS "inter-frame-spaces" logic was *way* overcomplicated (and fundamentally mistaken, though it happened to work) Now: - For TTS word-by-word output and `TTSSpeakFrames`: `TTSTextFrame`s' have `includes_inter_frame_spaces=False`. - For all other TTS output: `TTSTextFrame` pass through the received text frames' `includes_inter_frame_spaces` value. So far, this value has always been `True`: LLMs send text chunks already containing all necessary spaces. - `LLMTextFrame`s set `includes_inter_frame_spaces=False` at init time, per the aforementioned assumption. --- CHANGELOG.md | 8 +--- src/pipecat/frames/frames.py | 5 ++- src/pipecat/services/anthropic/llm.py | 4 +- src/pipecat/services/asyncai/tts.py | 18 -------- src/pipecat/services/aws/llm.py | 4 +- src/pipecat/services/aws/tts.py | 9 ---- src/pipecat/services/azure/tts.py | 9 ---- src/pipecat/services/deepgram/tts.py | 18 -------- src/pipecat/services/fish/tts.py | 9 ---- .../services/google/gemini_live/llm.py | 2 - src/pipecat/services/google/llm.py | 4 +- src/pipecat/services/google/tts.py | 18 -------- src/pipecat/services/groq/tts.py | 9 ---- src/pipecat/services/hume/tts.py | 9 ---- src/pipecat/services/inworld/tts.py | 9 ---- src/pipecat/services/lmnt/tts.py | 9 ---- src/pipecat/services/minimax/tts.py | 9 ---- src/pipecat/services/neuphonic/tts.py | 18 -------- src/pipecat/services/openai/base_llm.py | 4 +- src/pipecat/services/openai/realtime/llm.py | 2 - src/pipecat/services/openai/tts.py | 9 ---- src/pipecat/services/piper/tts.py | 9 ---- src/pipecat/services/rime/tts.py | 9 ---- src/pipecat/services/riva/tts.py | 9 ---- src/pipecat/services/sambanova/llm.py | 4 +- src/pipecat/services/sarvam/tts.py | 18 -------- src/pipecat/services/speechmatics/tts.py | 9 ---- src/pipecat/services/tts_service.py | 44 +++++++++---------- 28 files changed, 33 insertions(+), 255 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cda253c7..6a4a64f2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT service from ElevenLabs. -- Added a `TTSService.includes_inter_frame_spaces` property getter, so that TTS - services that subclass `TTSService` can indicate whether the text in the - `TTSTextFrame`s they push already contain any necessary inter-frame spaces. - ### Changed - Updated all STT and TTS services to use consistent error handling pattern with @@ -56,8 +52,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no ONNX dependency or added processing complexity. ## [0.0.94] - 2025-11-10 diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 6f48f79f7..ddb4e5a14 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -352,7 +352,10 @@ class TextFrame(DataFrame): class LLMTextFrame(TextFrame): """Text frame generated by LLM services.""" - pass + def __post_init__(self): + super().__post_init__() + # LLM services send text frames with all necessary spaces included + self.includes_inter_frame_spaces = True @dataclass diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index a5c9ee791..2e3e0272d 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -373,9 +373,7 @@ class AnthropicLLMService(LLMService): if event.type == "content_block_delta": if hasattr(event.delta, "text"): - frame = LLMTextFrame(event.delta.text) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self.push_frame(LLMTextFrame(event.delta.text)) completion_tokens_estimate += self._estimate_tokens(event.delta.text) elif hasattr(event.delta, "partial_json") and tool_use_block: json_accumulator += event.delta.partial_json diff --git a/src/pipecat/services/asyncai/tts.py b/src/pipecat/services/asyncai/tts.py index 8df597c56..f916d9bdd 100644 --- a/src/pipecat/services/asyncai/tts.py +++ b/src/pipecat/services/asyncai/tts.py @@ -146,15 +146,6 @@ class AsyncAITTSService(InterruptibleTTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that AsyncAI TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that AsyncAI's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Async language format. @@ -433,15 +424,6 @@ class AsyncAIHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that AsyncAI TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that AsyncAI's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Async language format. diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index 147a8c12a..ccbac43b7 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -1078,9 +1078,7 @@ class AWSBedrockLLMService(LLMService): if "contentBlockDelta" in event: delta = event["contentBlockDelta"]["delta"] if "text" in delta: - frame = LLMTextFrame(delta["text"]) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self.push_frame(LLMTextFrame(delta["text"])) completion_tokens_estimate += self._estimate_tokens(delta["text"]) elif "toolUse" in delta and "input" in delta["toolUse"]: # Handle partial JSON for tool use diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index cbc35b123..f22c42399 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -209,15 +209,6 @@ class AWSPollyTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that AWS TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that AWS's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to AWS Polly language format. diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index 8ac8b70e3..a1040f312 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -151,15 +151,6 @@ class AzureBaseTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Azure TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Azure's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Azure language format. diff --git a/src/pipecat/services/deepgram/tts.py b/src/pipecat/services/deepgram/tts.py index f6045c1f3..f75d40b09 100644 --- a/src/pipecat/services/deepgram/tts.py +++ b/src/pipecat/services/deepgram/tts.py @@ -79,15 +79,6 @@ class DeepgramTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Deepgram TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Deepgram's text frames include necessary inter-frame spaces. - """ - return True - @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Deepgram's TTS API. @@ -177,15 +168,6 @@ class DeepgramHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Deepgram TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Deepgram's text frames include necessary inter-frame spaces. - """ - return True - @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Deepgram's TTS API. diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index 0acb12a96..5fe129998 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -159,15 +159,6 @@ class FishAudioTTSService(InterruptibleTTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Fish Audio TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Fish Audio's text frames include necessary inter-frame spaces. - """ - return True - async def set_model(self, model: str): """Set the TTS model and reconnect. diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 11632968e..7e0b0f494 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -1452,8 +1452,6 @@ class GeminiLiveLLMService(LLMService): self._bot_text_buffer += text self._search_result_buffer += text # Also accumulate for grounding frame = LLMTextFrame(text=text) - # Gemini Live text already includes any necessary inter-chunk spaces - frame.includes_inter_frame_spaces = True await self.push_frame(frame) # Check for grounding metadata in server content diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index ad5fd70a7..883932b76 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -920,9 +920,7 @@ class GoogleLLMService(LLMService): for part in candidate.content.parts: if not part.thought and part.text: search_result += part.text - frame = LLMTextFrame(part.text) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self.push_frame(LLMTextFrame(part.text)) elif part.function_call: function_call = part.function_call id = function_call.id or str(uuid.uuid4()) diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index 449b2bca2..cf03e2d52 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -596,15 +596,6 @@ class GoogleHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Google TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Google's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Google TTS language format. @@ -803,15 +794,6 @@ class GoogleBaseTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Google and Gemini TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Google's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Google TTS language format. diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index d2dcb73a1..9026c4c4c 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -111,15 +111,6 @@ class GroqTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Groq TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Groq's text frames include necessary inter-frame spaces. - """ - return True - @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Groq's TTS API. diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 6760c8121..a3a7e9a4c 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -110,15 +110,6 @@ class HumeTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Hume TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Hume's text frames include necessary inter-frame spaces. - """ - return True - async def start(self, frame: StartFrame) -> None: """Start the service. diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index 067ea6ab6..dc2282b91 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -250,15 +250,6 @@ class InworldTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Inworld TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Inworld's text frames include necessary inter-frame spaces. - """ - return True - async def start(self, frame: StartFrame): """Start the Inworld TTS service. diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index d98d3f76c..ebcad0f20 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -124,15 +124,6 @@ class LmntTTSService(InterruptibleTTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that LMNT TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that LMNT's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to LMNT service language format. diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index b8e984eeb..05e2dac3b 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -194,15 +194,6 @@ class MiniMaxHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that MiniMax TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that MiniMax's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to MiniMax service language format. diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index 992abbe67..60b0ebcb1 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -151,15 +151,6 @@ class NeuphonicTTSService(InterruptibleTTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Neuphonic TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Neuphonic's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Neuphonic service language format. @@ -449,15 +440,6 @@ class NeuphonicHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Neuphonic TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Neuphonic's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Neuphonic service language format. diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 5a8c1ab31..d020e1106 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -390,9 +390,7 @@ class BaseOpenAILLMService(LLMService): # Keep iterating through the response to collect all the argument fragments arguments += tool_call.function.arguments elif chunk.choices[0].delta.content: - frame = LLMTextFrame(chunk.choices[0].delta.content) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self.push_frame(LLMTextFrame(chunk.choices[0].delta.content)) # When gpt-4o-audio / gpt-4o-mini-audio is used for llm or stt+llm # we need to get LLMTextFrame for the transcript diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index f66e6e8e1..8eaa3d6fa 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -678,8 +678,6 @@ class OpenAIRealtimeLLMService(LLMService): # the output modality is "text" if evt.delta: frame = LLMTextFrame(evt.delta) - # OpenAI Realtime text already includes any necessary inter-chunk spaces - frame.includes_inter_frame_spaces = True await self.push_frame(frame) async def _handle_evt_audio_transcript_delta(self, evt): diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index a5231170e..23cb75324 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -131,15 +131,6 @@ class OpenAITTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that OpenAI TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that OpenAI's text frames include necessary inter-frame spaces. - """ - return True - async def set_model(self, model: str): """Set the TTS model to use. diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index 3752418b5..dd842ff11 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -66,15 +66,6 @@ class PiperTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Piper TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Piper's text frames include necessary inter-frame spaces. - """ - return True - @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Piper's HTTP API. diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 329aecd7d..7b62f20fa 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -501,15 +501,6 @@ class RimeHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Rime TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Rime's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> str | None: """Convert pipecat language to Rime language code. diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py index 07aa4d105..6de6f9332 100644 --- a/src/pipecat/services/riva/tts.py +++ b/src/pipecat/services/riva/tts.py @@ -113,15 +113,6 @@ class RivaTTSService(TTSService): riva.client.proto.riva_tts_pb2.RivaSynthesisConfigRequest() ) - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Riva TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Riva's text frames include necessary inter-frame spaces. - """ - return True - async def set_model(self, model: str): """Attempt to set the TTS model. diff --git a/src/pipecat/services/sambanova/llm.py b/src/pipecat/services/sambanova/llm.py index 76f11e81c..5ed600457 100644 --- a/src/pipecat/services/sambanova/llm.py +++ b/src/pipecat/services/sambanova/llm.py @@ -176,9 +176,7 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore # Keep iterating through the response to collect all the argument fragments arguments += tool_call.function.arguments elif chunk.choices[0].delta.content: - frame = LLMTextFrame(chunk.choices[0].delta.content) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self.push_frame(LLMTextFrame(chunk.choices[0].delta.content)) # When gpt-4o-audio / gpt-4o-mini-audio is used for llm or stt+llm # we need to get LLMTextFrame for the transcript diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index 6ce45d27d..127a0d589 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -195,15 +195,6 @@ class SarvamHttpTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Sarvam TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Sarvam's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Sarvam AI language format. @@ -467,15 +458,6 @@ class SarvamTTSService(InterruptibleTTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Sarvam TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Sarvam's text frames include necessary inter-frame spaces. - """ - return True - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Sarvam AI language format. diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index e115d5a7c..b8fe172e7 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -105,15 +105,6 @@ class SpeechmaticsTTSService(TTSService): """ return True - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates that Speechmatics TTSTextFrames include necessary inter-frame spaces. - - Returns: - True, indicating that Speechmatics's text frames include necessary inter-frame spaces. - """ - return True - @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Speechmatics' HTTP API. diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 29c54f497..f0d602a40 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -142,6 +142,7 @@ class TTSService(AIService): self._voice_id: str = "" self._settings: Dict[str, Any] = {} self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() + self._aggregated_text_includes_inter_frame_spaces: bool = False self._text_filters: Sequence[BaseTextFilter] = text_filters or [] self._transport_destination: Optional[str] = transport_destination self._tracing_enabled: bool = False @@ -192,23 +193,6 @@ class TTSService(AIService): CHUNK_SECONDS = 0.5 return int(self.sample_rate * CHUNK_SECONDS * 2) # 2 bytes/sample - @property - def includes_inter_frame_spaces(self) -> bool: - """Indicates whether TTSTextFrames include necesary inter-frame spaces. - - When True, the TTSTextFrame objects pushed by this service already - include all necessary spaces between subsequent frames. When False, - downstream processors (like the assistant context aggregator) may need - to add spacing. - - Subclasses should override this property to return True if their text - generation process already includes necessary inter-frame spaces. - - Returns: - False by default. Subclasses can override to return True. - """ - return False - async def set_model(self, model: str): """Set the TTS model to use. @@ -369,9 +353,16 @@ class TTSService(AIService): await self._maybe_pause_frame_processing() sentence = self._text_aggregator.text + includes_inter_frame_spaces = self._aggregated_text_includes_inter_frame_spaces + + # Reset aggregator state await self._text_aggregator.reset() self._processing_text = False - await self._push_tts_frames(sentence) + self._aggregated_text_includes_inter_frame_spaces = False + + await self._push_tts_frames( + sentence, includes_inter_frame_spaces=includes_inter_frame_spaces + ) if isinstance(frame, LLMFullResponseEndFrame): if self._push_text_frames: await self.push_frame(frame, direction) @@ -380,7 +371,8 @@ class TTSService(AIService): elif isinstance(frame, TTSSpeakFrame): # Store if we were processing text or not so we can set it back. processing_text = self._processing_text - await self._push_tts_frames(frame.text) + # Assumption: text in TTSSpeakFrame does not include inter-frame spaces + await self._push_tts_frames(frame.text, includes_inter_frame_spaces=False) # We pause processing incoming frames because we are sending data to # the TTS. We pause to avoid audio overlapping. await self._maybe_pause_frame_processing() @@ -474,11 +466,17 @@ class TTSService(AIService): text = frame.text else: text = await self._text_aggregator.aggregate(frame.text) + # Assumption: whether inter-frame spaces are included shouldn't + # change during aggregation, so we can just use the latest frame's + # value + self._aggregated_text_includes_inter_frame_spaces = frame.includes_inter_frame_spaces if text: - await self._push_tts_frames(text) + await self._push_tts_frames( + text, includes_inter_frame_spaces=frame.includes_inter_frame_spaces + ) - async def _push_tts_frames(self, text: str): + async def _push_tts_frames(self, text: str, includes_inter_frame_spaces: bool): # Remove leading newlines only text = text.lstrip("\n") @@ -508,7 +506,7 @@ class TTSService(AIService): # We send the original text after the audio. This way, if we are # interrupted, the text is not added to the assistant context. frame = TTSTextFrame(text) - frame.includes_inter_frame_spaces = self.includes_inter_frame_spaces + frame.includes_inter_frame_spaces = includes_inter_frame_spaces await self.push_frame(frame) async def _stop_frame_handler(self): @@ -635,6 +633,8 @@ class WordTTSService(TTSService): frame = TTSStoppedFrame() frame.pts = last_pts else: + # Assumption: word-by-word text frames don't include spaces, so + # we can rely on the default includes_inter_frame_spaces=False frame = TTSTextFrame(word) frame.pts = self._initial_word_timestamp + timestamp if frame: From 0c3c26b7b87863ecc79f148ad394adc4f939a31b Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 15:20:09 -0300 Subject: [PATCH 07/13] Passing the custom request_data to the SmallWebRTCRunnerArguments body. --- src/pipecat/runner/run.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index 55c70ed8a..ebca467df 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -264,7 +264,10 @@ def _setup_webrtc_routes( # Prepare runner arguments with the callback to run your bot async def webrtc_connection_callback(connection): bot_module = _get_bot_module() - runner_args = SmallWebRTCRunnerArguments(webrtc_connection=connection) + + runner_args = SmallWebRTCRunnerArguments( + webrtc_connection=connection, body=request.request_data + ) background_tasks.add_task(bot_module.bot, runner_args) # Delegate handling to SmallWebRTCRequestHandler @@ -326,7 +329,8 @@ def _setup_webrtc_routes( type=request_data["type"], pc_id=request_data.get("pc_id"), restart_pc=request_data.get("restart_pc"), - request_data=request_data, + request_data=request_data.get("request_data") + or request_data.get("requestData"), ) return await offer(webrtc_request, background_tasks) elif request.method == HTTPMethod.PATCH.value: From 74154b26a2a97840bd60a4f8ddb676da7a26f77f Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 17 Nov 2025 15:39:07 -0300 Subject: [PATCH 08/13] Mentioning the SmallWebRTCTransport fix in the readme. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33f7fdc6b..bca4a5d6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue in the `Runner` where, when using `SmallWebRTCTransport`, the + `request_data` was not being passed to the `SmallWebRTCRunnerArguments` body. + - Fixed subtle issue of assistant context messages ending up with double spaces between words or sentences. From 7eedb33d5076de8584452dd311c844bb46122cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 17 Nov 2025 11:19:04 -0800 Subject: [PATCH 09/13] BaseTextFilter: only require subclasses to implement filter() --- CHANGELOG.md | 12 ++++++------ src/pipecat/utils/text/base_text_filter.py | 3 --- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bca4a5d6f..d4fce0917 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `ElevenLabsRealtimeSTTService` which implements the Realtime STT service from ElevenLabs. +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no + ONNX dependency or added processing complexity. + ### Changed +- `BaseTextFilter` only require subclasses to implement the `filter()` method. + - Extracted the logic for retrying connections, and create a new `send_with_retry` method inside `WebSocketService`. @@ -65,12 +71,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. -### Added - -- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and - example wiring; leverages the enhancement model for robust detection with no - ONNX dependency or added processing complexity. - ## [0.0.94] - 2025-11-10 ### Changed diff --git a/src/pipecat/utils/text/base_text_filter.py b/src/pipecat/utils/text/base_text_filter.py index 1a18a38a6..0ede2137c 100644 --- a/src/pipecat/utils/text/base_text_filter.py +++ b/src/pipecat/utils/text/base_text_filter.py @@ -26,7 +26,6 @@ class BaseTextFilter(ABC): behavior, settings management, and interruption handling logic. """ - @abstractmethod async def update_settings(self, settings: Mapping[str, Any]): """Update the filter's configuration settings. @@ -53,7 +52,6 @@ class BaseTextFilter(ABC): """ pass - @abstractmethod async def handle_interruption(self): """Handle interruption events in the processing pipeline. @@ -62,7 +60,6 @@ class BaseTextFilter(ABC): """ pass - @abstractmethod async def reset_interruption(self): """Reset the filter state after an interruption has been handled. From 5095fc6a64da534abfedab0c438761288547087e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Nov 2025 15:16:19 -0500 Subject: [PATCH 10/13] Update Moondream example so that Moondream service output makes it into the context, even if the TTS service is disabled --- .../14d-function-calling-moondream-video.py | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/examples/foundational/14d-function-calling-moondream-video.py b/examples/foundational/14d-function-calling-moondream-video.py index 6aeb2b892..9544818b9 100644 --- a/examples/foundational/14d-function-calling-moondream-video.py +++ b/examples/foundational/14d-function-calling-moondream-video.py @@ -15,14 +15,21 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame +from pipecat.frames.frames import ( + Frame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMRunFrame, + TextFrame, + UserImageRequestFrame, +) from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.processors.frame_processor import FrameDirection +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -66,6 +73,27 @@ async def fetch_user_image(params: FunctionCallParams): # await params.result_callback({"result": "Image is being captured."}) +class MoondreamTextFrameWrapper(FrameProcessor): + """Wraps Moondream-provided TextFrames with LLM response start/end frames. + + This processor detects TextFrames and automatically wraps them with + LLMFullResponseStartFrame and LLMFullResponseEndFrame to provide proper + response boundaries for downstream processors. + """ + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + # If we receive a TextFrame, wrap it with response start/end frames + if isinstance(frame, TextFrame): + await self.push_frame(LLMFullResponseStartFrame(), direction) + await self.push_frame(frame, direction) + await self.push_frame(LLMFullResponseEndFrame(), direction) + else: + # For all other frames, just pass them through + await self.push_frame(frame, direction) + + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -130,6 +158,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # If you run into weird description, try with use_cpu=True moondream = MoondreamService() + # Wrap TextFrames with LLM response start/end frames, which makes Moondream + # output be treated like LLM responses for the purpose of context + # aggregation. Without this, the assistant context aggregator would ignore + # Moondream output (if the TTS service is disabled). + moondream_text_wrapper = MoondreamTextFrameWrapper() + pipeline = Pipeline( [ transport.input(), # Transport user input @@ -137,7 +171,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context_aggregator.user(), # User responses ParallelPipeline( [llm], # LLM - [moondream], + [moondream, moondream_text_wrapper], ), tts, # TTS transport.output(), # Transport bot output From 4835617b16a1f4a5fbbb2f70562454c75a568573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Nov 2025 15:20:01 -0800 Subject: [PATCH 11/13] ConsumerProcessor: queue frames internally instead of pushing them --- CHANGELOG.md | 4 ++++ src/pipecat/processors/consumer_processor.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4fce0917..04d789370 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `ConsumerProcessor` now queues frames from the producer internally instead of + pushing them directly. This allows us to subclass consumer processors and + manipulate frames before they are pushed. + - `BaseTextFilter` only require subclasses to implement the `filter()` method. - Extracted the logic for retrying connections, and create a new `send_with_retry` diff --git a/src/pipecat/processors/consumer_processor.py b/src/pipecat/processors/consumer_processor.py index 5445b492d..3654194ec 100644 --- a/src/pipecat/processors/consumer_processor.py +++ b/src/pipecat/processors/consumer_processor.py @@ -83,4 +83,4 @@ class ConsumerProcessor(FrameProcessor): while True: frame = await self._queue.get() new_frame = await self._transformer(frame) - await self.push_frame(new_frame, self._direction) + await self.queue_frame(new_frame, self._direction) From 3132e1226571228bda6fc6a323128a2c3095e003 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 14 Nov 2025 10:48:52 -0500 Subject: [PATCH 12/13] Add camera and screen capture support to dev runner for SmallWebRTC --- CHANGELOG.md | 8 ++++++++ src/pipecat/runner/utils.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04d789370..de1f520a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no + ONNX dependency or added processing complexity. + - Added a watchdog to `DeepgramFluxSTTService` to prevent dangling tasks in case the user was speaking and we stop receiving audio. @@ -39,6 +43,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated all STT and TTS services to use consistent error handling pattern with `push_error()` method for better pipeline error event integration. +- Added support for `maybe_capture_participant_camera()` and + `maybe_capture_participant_screen()` for `SmallWebRTCTransport` in the runner + utils. + - Added Hindi support for Rime TTS services. - Updated `GeminiTTSService` to use Google Cloud Text-to-Speech streaming API diff --git a/src/pipecat/runner/utils.py b/src/pipecat/runner/utils.py index f9fd0c14a..76a6fa82f 100644 --- a/src/pipecat/runner/utils.py +++ b/src/pipecat/runner/utils.py @@ -281,6 +281,14 @@ async def maybe_capture_participant_camera( except ImportError: pass + try: + from pipecat.transports.smallwebrtc.transport import SmallWebRTCTransport + + if isinstance(transport, SmallWebRTCTransport): + await transport.capture_participant_video(video_source="camera") + except ImportError: + pass + async def maybe_capture_participant_screen( transport: BaseTransport, client: Any, framerate: int = 0 @@ -303,6 +311,14 @@ async def maybe_capture_participant_screen( except ImportError: pass + try: + from pipecat.transports.smallwebrtc.transport import SmallWebRTCTransport + + if isinstance(transport, SmallWebRTCTransport): + await transport.capture_participant_video(video_source="screenVideo") + except ImportError: + pass + def _smallwebrtc_sdp_cleanup_ice_candidates(text: str, pattern: str) -> str: """Clean up ICE candidates in SDP text for SmallWebRTC. From 9f45ad4d2e2ec9713a95c9d7c39b2bba5dd258cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Nov 2025 08:38:45 -0800 Subject: [PATCH 13/13] LLMContext: create_image_message/create_audio_message are now async --- CHANGELOG.md | 5 ++ .../foundational/12-describe-image-openai.py | 2 +- .../12a-describe-image-anthropic.py | 2 +- .../foundational/12b-describe-image-aws.py | 2 +- .../12c-describe-image-gemini-flash.py | 2 +- .../processors/aggregators/llm_context.py | 46 ++++++++++++------- 6 files changed, 38 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de1f520a7..82da28b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- ⚠️ Breaking change: `LLMContext.create_image_message()` and + `LLMContext.create_audio_message()` are now async methods. This fixes and + issue where the asyncio event loop would be blocked while encoding audio or + images. + - `ConsumerProcessor` now queues frames from the producer internally instead of pushing them directly. This allows us to subclass consumer processors and manipulate frames before they are pushed. diff --git a/examples/foundational/12-describe-image-openai.py b/examples/foundational/12-describe-image-openai.py index 8c72075e8..477803da6 100644 --- a/examples/foundational/12-describe-image-openai.py +++ b/examples/foundational/12-describe-image-openai.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Kick off the conversation. image = Image.open(image_path) - message = LLMContext.create_image_message( + message = await LLMContext.create_image_message( image=image.tobytes(), format="RGB", size=image.size, diff --git a/examples/foundational/12a-describe-image-anthropic.py b/examples/foundational/12a-describe-image-anthropic.py index 6a9891712..ac4e8f01c 100644 --- a/examples/foundational/12a-describe-image-anthropic.py +++ b/examples/foundational/12a-describe-image-anthropic.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Kick off the conversation. image = Image.open(image_path) - message = LLMContext.create_image_message( + message = await LLMContext.create_image_message( image=image.tobytes(), format="RGB", size=image.size, diff --git a/examples/foundational/12b-describe-image-aws.py b/examples/foundational/12b-describe-image-aws.py index 441c49cfd..cf1ce66a0 100644 --- a/examples/foundational/12b-describe-image-aws.py +++ b/examples/foundational/12b-describe-image-aws.py @@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Kick off the conversation. image = Image.open(image_path) - message = LLMContext.create_image_message( + message = await LLMContext.create_image_message( image=image.tobytes(), format="RGB", size=image.size, diff --git a/examples/foundational/12c-describe-image-gemini-flash.py b/examples/foundational/12c-describe-image-gemini-flash.py index 919bf3553..bfd7f5146 100644 --- a/examples/foundational/12c-describe-image-gemini-flash.py +++ b/examples/foundational/12c-describe-image-gemini-flash.py @@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Kick off the conversation. image = Image.open(image_path) - message = LLMContext.create_image_message( + message = await LLMContext.create_image_message( image=image.tobytes(), format="RGB", size=image.size, diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index d9280f9c0..b9216103a 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -14,6 +14,7 @@ translation from this universal context into whatever format it needs, using a service-specific adapter. """ +import asyncio import base64 import io import wave @@ -137,7 +138,7 @@ class LLMContext: return {"role": role, "content": content} @staticmethod - def create_image_message( + async def create_image_message( *, role: str = "user", format: str, @@ -154,15 +155,21 @@ class LLMContext: image: Raw image bytes. text: Optional text to include with the image. """ - buffer = io.BytesIO() - Image.frombytes(format, size, image).save(buffer, format="JPEG") - encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") + + def encode_image(): + buffer = io.BytesIO() + Image.frombytes(format, size, image).save(buffer, format="JPEG") + encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") + return encoded_image + + encoded_image = await asyncio.to_thread(encode_image) + url = f"data:image/jpeg;base64,{encoded_image}" return LLMContext.create_image_url_message(role=role, url=url, text=text) @staticmethod - def create_audio_message( + async def create_audio_message( *, role: str = "user", audio_frames: list[AudioRawFrame], text: str = "Audio follows" ) -> LLMContextMessage: """Create a context message containing audio. @@ -172,21 +179,26 @@ class LLMContext: audio_frames: List of audio frame objects to include. text: Optional text to include with the audio. """ - sample_rate = audio_frames[0].sample_rate - num_channels = audio_frames[0].num_channels - content = [] - content.append({"type": "text", "text": text}) - data = b"".join(frame.audio for frame in audio_frames) + def encode_audio(): + sample_rate = audio_frames[0].sample_rate + num_channels = audio_frames[0].num_channels - with io.BytesIO() as buffer: - with wave.open(buffer, "wb") as wf: - wf.setsampwidth(2) - wf.setnchannels(num_channels) - wf.setframerate(sample_rate) - wf.writeframes(data) + content = [] + content.append({"type": "text", "text": text}) + data = b"".join(frame.audio for frame in audio_frames) - encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") + with io.BytesIO() as buffer: + with wave.open(buffer, "wb") as wf: + wf.setsampwidth(2) + wf.setnchannels(num_channels) + wf.setframerate(sample_rate) + wf.writeframes(data) + + encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") + return encoded_audio + + encoded_audio = asyncio.to_thread(encode_audio) content.append( {