Merge branch 'main' into feat/add-cached-reasoning-tokens-metrics-to-opentel-spans

src/pipecat/audio/filters/aic_filter.py

@@ -68,6 +68,58 @@ class AICFilter(BaseAudioFilter):
         # Model will be created in start() since the API now requires sample_rate
         self._aic = None
 
+    def get_vad_factory(self):
+        """Return a zero-arg factory that will create the VAD once the model exists.
+
+        Returns:
+            A zero-argument callable that, when invoked, returns an initialized
+            VoiceActivityDetector bound to the underlying AIC model. Raises a
+            RuntimeError if the model has not been initialized (i.e. start()
+            has not been called successfully).
+        """
+
+        def _factory():
+            if self._aic is None:
+                raise RuntimeError("AIC model not initialized yet. Call start(sample_rate) first.")
+            return self._aic.create_vad()
+
+        return _factory
+
+    def create_vad_analyzer(
+        self,
+        *,
+        lookback_buffer_size: Optional[float] = None,
+        sensitivity: Optional[float] = None,
+    ):
+        """Return an analyzer that will lazily instantiate the AIC VAD when ready.
+
+        AIC VAD parameters:
+          - lookback_buffer_size:
+              Number of window-length audio buffers used as a lookback buffer.
+              Higher values increase prediction stability but add latency.
+              Range: 1.0 .. 20.0, Default (SDK): 6.0
+          - sensitivity:
+              Energy threshold sensitivity. Energy threshold = 10 ** (-sensitivity).
+              Range: 1.0 .. 15.0, Default (SDK): 6.0
+
+        Args:
+            lookback_buffer_size: Optional lookback buffer size to configure on the VAD.
+                Range: 1.0 .. 20.0. If None, SDK default is used.
+            sensitivity: Optional sensitivity (energy threshold) to configure on the VAD.
+                Range: 1.0 .. 15.0. If None, SDK default is used.
+
+        Returns:
+            A lazily-initialized AICVADAnalyzer that will bind to the VAD backend
+            once the filter's model has been created (after start(sample_rate)).
+        """
+        from pipecat.audio.vad.aic_vad import AICVADAnalyzer
+
+        return AICVADAnalyzer(
+            vad_factory=self.get_vad_factory(),
+            lookback_buffer_size=lookback_buffer_size,
+            sensitivity=sensitivity,
+        )
+
     async def start(self, sample_rate: int):
         """Initialize the filter with the transport's sample rate.
 
@@ -185,7 +237,7 @@ class AICFilter(BaseAudioFilter):
             )
 
             # Process planar in-place; returns ndarray (same shape)
-            out_f32 = self._aic.process(block_f32)
+            out_f32 = await self._aic.process_async(block_f32)
 
             # Convert back to int16 bytes, planar layout
             out_i16 = np.clip(out_f32 * 32768.0, -32768, 32767).astype(np.int16)

src/pipecat/audio/filters/krisp_filter.py

@@ -61,6 +61,10 @@ class KrispFilter(BaseAudioFilter):
     Provides real-time noise reduction for audio streams using Krisp's
     proprietary noise suppression algorithms. Requires a Krisp model file
     for operation.
+
+    .. deprecated:: 0.0.94
+        The KrispFilter is deprecated and will be removed in a future version.
+        Use KrispVivaFilter instead.
     """
 
     def __init__(
@@ -79,6 +83,17 @@ class KrispFilter(BaseAudioFilter):
         """
         super().__init__()
 
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "KrispFilter is deprecated and will be removed in a future version. "
+                "Use KrispVivaFilter instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
         # Set model path, checking environment if not specified
         self._model_path = model_path or os.getenv("KRISP_MODEL_PATH")
         if not self._model_path:

src/pipecat/audio/vad/aic_vad.py

@@ -0,0 +1,158 @@
+"""AIC-integrated VAD analyzer that lazily binds to the AIC SDK backend.
+
+This analyzer queries the backend's is_speech_detected() and maps it to a float
+confidence (1.0/0.0). It uses 10 ms windows based on the sample rate and applies
+optional AIC VAD parameters (lookback_buffer_size, sensitivity) when available.
+"""
+
+from typing import Any, Callable, Optional
+
+from loguru import logger
+
+from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams
+
+try:
+    from aic import AICVadParameter
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.")
+    raise Exception(f"Missing module: {e}")
+
+
+class AICVADAnalyzer(VADAnalyzer):
+    """VAD analyzer that lazily instantiates the AIC VoiceActivityDetector via a factory.
+
+    The analyzer can be constructed before the AIC Model exists. Once the filter has
+    started and the Model is available, the provided factory will succeed and the
+    backend VAD will be created. We then switch to single-sample updates where
+    num_frames_required() returns 1 and confidence is derived from the backend's
+    boolean is_speech_detected() state.
+
+    AIC VAD runtime parameters:
+      - lookback_buffer_size:
+          Controls the lookback buffer size used by the VAD, i.e. the number of
+          window-length audio buffers used as a lookback buffer. Larger values improve
+          stability but increase latency.
+          Range: 1.0 .. 20.0
+          Default (SDK): 6.0
+      - sensitivity:
+          Controls the energy threshold sensitivity. Higher values make the detector
+          less sensitive (require more energy to count as speech).
+          Range: 1.0 .. 15.0
+          Formula: Energy threshold = 10 ** (-sensitivity)
+          Default (SDK): 6.0
+    """
+
+    def __init__(
+        self,
+        *,
+        vad_factory: Optional[Callable[[], Any]] = None,
+        lookback_buffer_size: Optional[float] = None,
+        sensitivity: Optional[float] = None,
+    ):
+        """Create an AIC VAD analyzer.
+
+        Args:
+            vad_factory:
+                Zero-arg callable that returns an initialized AIC VoiceActivityDetector.
+                This may raise until the filter's Model has been created; the analyzer
+                will retry on set_sample_rate/first use.
+            lookback_buffer_size:
+                Optional override for AIC VAD lookback buffer size.
+                Range: 1.0 .. 20.0. Larger values increase stability at the cost of latency.
+                If None, the SDK default (6.0) is used.
+            sensitivity:
+                Optional override for AIC VAD sensitivity (energy threshold).
+                Range: 1.0 .. 15.0. Energy threshold = 10 ** (-sensitivity).
+                If None, the SDK default (6.0) is used.
+        """
+        # Use fixed VAD parameters for AIC: no user override
+        fixed_params = VADParams(confidence=0.5, start_secs=0.0, stop_secs=0.0, min_volume=0.0)
+        super().__init__(sample_rate=None, params=fixed_params)
+        self._vad_factory = vad_factory
+        self._backend_vad: Optional[Any] = None
+        self._pending_lookback: Optional[float] = lookback_buffer_size
+        self._pending_sensitivity: Optional[float] = sensitivity
+
+    def bind_vad_factory(self, vad_factory: Callable[[], Any]):
+        """Attach or replace the factory post-construction."""
+        self._vad_factory = vad_factory
+        self._ensure_backend_initialized()
+
+    def _apply_backend_params(self):
+        """Apply optional AIC VAD parameters if available."""
+        if self._backend_vad is None or AICVadParameter is None:
+            return
+        try:
+            if self._pending_lookback is not None:
+                self._backend_vad.set_parameter(
+                    AICVadParameter.LOOKBACK_BUFFER_SIZE, float(self._pending_lookback)
+                )
+            if self._pending_sensitivity is not None:
+                self._backend_vad.set_parameter(
+                    AICVadParameter.SENSITIVITY, float(self._pending_sensitivity)
+                )
+        except Exception as e:  # noqa: BLE001
+            logger.debug(f"AIC VAD parameter application deferred/failed: {e}")
+
+    def _ensure_backend_initialized(self):
+        if self._backend_vad is not None:
+            return
+        if not self._vad_factory:
+            return
+        try:
+            self._backend_vad = self._vad_factory()
+            self._apply_backend_params()
+            # With backend ready, recompute internal frame sizing
+            super().set_params(self._params)
+            logger.debug("AIC VAD backend initialized in analyzer.")
+        except Exception as e:  # noqa: BLE001
+            # Filter may not be started yet; try again later
+            logger.debug(f"Deferring AIC VAD backend initialization: {e}")
+
+    def set_sample_rate(self, sample_rate: int):
+        """Set the sample rate for audio processing.
+
+        Args:
+            sample_rate: Audio sample rate in Hz.
+        """
+        # Set rate and attempt backend initialization once we know SR
+        self._sample_rate = self._init_sample_rate or sample_rate
+        self._ensure_backend_initialized()
+        # Ensure params are initialized even if backend not ready yet
+        try:
+            super().set_params(self._params)
+        except Exception:
+            pass
+
+    def num_frames_required(self) -> int:
+        """Get the number of audio frames required for analysis.
+
+        Returns:
+            Number of frames needed for VAD processing.
+        """
+        # Use 10 ms windows based on sample rate
+        return int(self.sample_rate * 0.01) if self.sample_rate > 0 else 160
+
+    def voice_confidence(self, buffer: bytes) -> float:
+        """Calculate voice activity confidence for the given audio buffer.
+
+        Args:
+            buffer: Audio buffer to analyze.
+
+        Returns:
+            Voice confidence score is 0.0 or 1.0.
+        """
+        # Ensure backend exists (filter might have started since last call)
+        self._ensure_backend_initialized()
+        if self._backend_vad is None:
+            return 0.0
+
+        # We do not need to analyze 'buffer' here since the model's VAD is updated
+        # as part of the enhancement pipeline. Simply query the boolean and map it.
+        try:
+            is_speech = self._backend_vad.is_speech_detected()
+            return 1.0 if is_speech else 0.0
+        except Exception as e:  # noqa: BLE001
+            logger.error(f"AIC VAD inference error: {e}")
+            return 0.0

src/pipecat/extensions/ivr/ivr_navigator.py

@@ -31,7 +31,11 @@ from pipecat.pipeline.pipeline import Pipeline
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.services.llm_service import LLMService
-from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator
+from pipecat.utils.text.pattern_pair_aggregator import (
+    MatchAction,
+    PatternMatch,
+    PatternPairAggregator,
+)
 
 
 class IVRStatus(Enum):
@@ -114,15 +118,15 @@ class IVRProcessor(FrameProcessor):
     def _setup_xml_patterns(self):
         """Set up XML pattern detection and handlers."""
         # Register DTMF pattern
-        self._aggregator.add_pattern_pair("dtmf", "<dtmf>", "</dtmf>", remove_match=True)
+        self._aggregator.add_pattern("dtmf", "<dtmf>", "</dtmf>", action=MatchAction.REMOVE)
         self._aggregator.on_pattern_match("dtmf", self._handle_dtmf_action)
 
         # Register mode pattern
-        self._aggregator.add_pattern_pair("mode", "<mode>", "</mode>", remove_match=True)
+        self._aggregator.add_pattern("mode", "<mode>", "</mode>", action=MatchAction.REMOVE)
         self._aggregator.on_pattern_match("mode", self._handle_mode_action)
 
         # Register IVR pattern
-        self._aggregator.add_pattern_pair("ivr", "<ivr>", "</ivr>", remove_match=True)
+        self._aggregator.add_pattern("ivr", "<ivr>", "</ivr>", action=MatchAction.REMOVE)
         self._aggregator.on_pattern_match("ivr", self._handle_ivr_action)
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -148,7 +152,7 @@ class IVRProcessor(FrameProcessor):
             result = await self._aggregator.aggregate(frame.text)
             if result:
                 # Push aggregated text that doesn't contain XML patterns
-                await self.push_frame(LLMTextFrame(result), direction)
+                await self.push_frame(LLMTextFrame(result.text), direction)
 
         else:
             await self.push_frame(frame, direction)
@@ -159,7 +163,7 @@ class IVRProcessor(FrameProcessor):
         Args:
             match: The pattern match containing DTMF content.
         """
-        value = match.content
+        value = match.text
         logger.debug(f"DTMF detected: {value}")
 
         try:
@@ -180,7 +184,7 @@ class IVRProcessor(FrameProcessor):
         Args:
             match: The pattern match containing IVR status content.
         """
-        status = match.content
+        status = match.text
         logger.trace(f"IVR status detected: {status}")
 
         # Convert string to enum, with validation
@@ -211,7 +215,7 @@ class IVRProcessor(FrameProcessor):
         Args:
             match: The pattern match containing mode content.
         """
-        mode = match.content
+        mode = match.text
         logger.debug(f"Mode detected: {mode}")
         if mode == "conversation":
             await self._handle_conversation()

src/pipecat/frames/frames.py

@@ -12,6 +12,7 @@ and LLM processing.
 """
 
 from dataclasses import dataclass, field
+from enum import Enum
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -330,10 +331,21 @@ class TextFrame(DataFrame):
 
     text: str
     skip_tts: bool = field(init=False)
+    # Whether any necessary inter-frame (leading/trailing) spaces are already
+    # included in the text.
+    # NOTE: Ideally this would be available at init time with a default value,
+    # but that would impact how subclasses can be initialized (it would require
+    # mandatory fields of theirs to have defaults to preserve
+    # non-default-before-default argument order)
+    includes_inter_frame_spaces: bool = field(init=False)
+    # Whether this text frame should be appended to the LLM context.
+    append_to_context: bool = field(init=False)
 
     def __post_init__(self):
         super().__post_init__()
         self.skip_tts = False
+        self.includes_inter_frame_spaces = False
+        self.append_to_context = True
 
     def __str__(self):
         pts = format_pts(self.pts)
@@ -344,11 +356,38 @@ class TextFrame(DataFrame):
 class LLMTextFrame(TextFrame):
     """Text frame generated by LLM services."""
 
-    pass
+    def __post_init__(self):
+        super().__post_init__()
+        # LLM services send text frames with all necessary spaces included
+        self.includes_inter_frame_spaces = True
+
+
+class AggregationType(str, Enum):
+    """Built-in aggregation strings."""
+
+    SENTENCE = "sentence"
+    WORD = "word"
+
+    def __str__(self):
+        return self.value
 
 
 @dataclass
-class TTSTextFrame(TextFrame):
+class AggregatedTextFrame(TextFrame):
+    """Text frame representing an aggregation of TextFrames.
+
+    This frame contains multiple TextFrames aggregated together for processing
+    or output along with a field to indicate how they are aggregated.
+
+    Parameters:
+        aggregated_by: Method used to aggregate the text frames.
+    """
+
+    aggregated_by: AggregationType | str
+
+
+@dataclass
+class TTSTextFrame(AggregatedTextFrame):
     """Text frame generated by Text-to-Speech services."""
 
     pass

src/pipecat/processors/aggregators/llm_context.py

@@ -14,6 +14,7 @@ translation from this universal context into whatever format it needs, using a
 service-specific adapter.
 """
 
+import asyncio
 import base64
 import io
 import wave
@@ -137,7 +138,7 @@ class LLMContext:
         return {"role": role, "content": content}
 
     @staticmethod
-    def create_image_message(
+    async def create_image_message(
         *,
         role: str = "user",
         format: str,
@@ -154,15 +155,21 @@ class LLMContext:
             image: Raw image bytes.
             text: Optional text to include with the image.
         """
-        buffer = io.BytesIO()
-        Image.frombytes(format, size, image).save(buffer, format="JPEG")
-        encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
+
+        def encode_image():
+            buffer = io.BytesIO()
+            Image.frombytes(format, size, image).save(buffer, format="JPEG")
+            encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
+            return encoded_image
+
+        encoded_image = await asyncio.to_thread(encode_image)
+
         url = f"data:image/jpeg;base64,{encoded_image}"
 
         return LLMContext.create_image_url_message(role=role, url=url, text=text)
 
     @staticmethod
-    def create_audio_message(
+    async def create_audio_message(
         *, role: str = "user", audio_frames: list[AudioRawFrame], text: str = "Audio follows"
     ) -> LLMContextMessage:
         """Create a context message containing audio.
@@ -172,21 +179,26 @@ class LLMContext:
             audio_frames: List of audio frame objects to include.
             text: Optional text to include with the audio.
         """
-        sample_rate = audio_frames[0].sample_rate
-        num_channels = audio_frames[0].num_channels
 
-        content = []
-        content.append({"type": "text", "text": text})
-        data = b"".join(frame.audio for frame in audio_frames)
+        async def encode_audio():
+            sample_rate = audio_frames[0].sample_rate
+            num_channels = audio_frames[0].num_channels
 
-        with io.BytesIO() as buffer:
-            with wave.open(buffer, "wb") as wf:
-                wf.setsampwidth(2)
-                wf.setnchannels(num_channels)
-                wf.setframerate(sample_rate)
-                wf.writeframes(data)
+            content = []
+            content.append({"type": "text", "text": text})
+            data = b"".join(frame.audio for frame in audio_frames)
 
-        encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8")
+            with io.BytesIO() as buffer:
+                with wave.open(buffer, "wb") as wf:
+                    wf.setsampwidth(2)
+                    wf.setnchannels(num_channels)
+                    wf.setframerate(sample_rate)
+                    wf.writeframes(data)
+
+            encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8")
+            return encoded_audio
+
+        encoded_audio = await asyncio.to_thread(encode_audio)
 
         content.append(
             {
@@ -321,7 +333,7 @@ class LLMContext:
         """
         self._tool_choice = tool_choice
 
-    def add_image_frame_message(
+    async def add_image_frame_message(
         self, *, format: str, size: tuple[int, int], image: bytes, text: Optional[str] = None
     ):
         """Add a message containing an image frame.
@@ -332,10 +344,12 @@ class LLMContext:
             image: Raw image bytes.
             text: Optional text to include with the image.
         """
-        message = LLMContext.create_image_message(format=format, size=size, image=image, text=text)
+        message = await LLMContext.create_image_message(
+            format=format, size=size, image=image, text=text
+        )
         self.add_message(message)
 
-    def add_audio_frames_message(
+    async def add_audio_frames_message(
         self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows"
     ):
         """Add a message containing audio frames.
@@ -344,7 +358,7 @@ class LLMContext:
             audio_frames: List of audio frame objects to include.
             text: Optional text to include with the audio.
         """
-        message = LLMContext.create_audio_message(audio_frames=audio_frames, text=text)
+        message = await LLMContext.create_audio_message(audio_frames=audio_frames, text=text)
         self.add_message(message)
 
     @staticmethod

src/pipecat/processors/aggregators/llm_response.py

@@ -1001,7 +1001,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
         await self.push_aggregation()
 
     async def _handle_text(self, frame: TextFrame):
-        if not self._started:
+        if not self._started or not frame.append_to_context:
             return
 
         if self._params.expect_stripped_words:

src/pipecat/processors/aggregators/llm_response_universal.py

@@ -66,7 +66,7 @@ from pipecat.processors.aggregators.llm_response import (
     LLMUserAggregatorParams,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.utils.string import concatenate_aggregated_text
+from pipecat.utils.string import TextPartForConcatenation, concatenate_aggregated_text
 from pipecat.utils.time import time_now_iso8601
 
 
@@ -90,7 +90,7 @@ class LLMContextAggregator(FrameProcessor):
         self._context = context
         self._role = role
 
-        self._aggregation: List[str] = []
+        self._aggregation: List[TextPartForConcatenation] = []
 
     @property
     def messages(self) -> List[LLMContextMessage]:
@@ -433,7 +433,12 @@ class LLMUserAggregator(LLMContextAggregator):
         if not text.strip():
             return
 
-        self._aggregation.append(text)
+        # Transcriptions never include inter-part spaces (so far).
+        self._aggregation.append(
+            TextPartForConcatenation(
+                text, includes_inter_part_spaces=frame.includes_inter_frame_spaces
+            )
+        )
         # We just got a final result, so let's reset interim results.
         self._seen_interim_results = False
         # Reset aggregation timer.
@@ -788,7 +793,7 @@ class LLMAssistantAggregator(LLMContextAggregator):
 
         logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})")
 
-        self._context.add_image_frame_message(
+        await self._context.add_image_frame_message(
             format=frame.format,
             size=frame.size,
             image=frame.image,
@@ -806,14 +811,18 @@ class LLMAssistantAggregator(LLMContextAggregator):
         await self.push_aggregation()
 
     async def _handle_text(self, frame: TextFrame):
-        if not self._started:
+        if not self._started or not frame.append_to_context:
             return
 
         # Make sure we really have text (spaces count, too!)
         if len(frame.text) == 0:
             return
 
-        self._aggregation.append(frame.text)
+        self._aggregation.append(
+            TextPartForConcatenation(
+                frame.text, includes_inter_part_spaces=frame.includes_inter_frame_spaces
+            )
+        )
 
     def _context_updated_task_finished(self, task: asyncio.Task):
         self._context_updated_tasks.discard(task)

src/pipecat/processors/aggregators/llm_text_processor.py

@@ -0,0 +1,106 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""LLM text processor module for processing and aggregating raw LLM output text.
+
+This processor will convert LLMTextFrames into AggregatedTextFrames based on the
+configured text aggregator. Using the customizable aggregator, it provides
+functionality to handle or manipulate LLM text frames before they are sent to other
+components such as TTS services or context aggregators. It can be used to pre-aggregate
+and categorize, modify, or filter direct output tokens from the LLM.
+"""
+
+from typing import Optional
+
+from pipecat.frames.frames import (
+    AggregatedTextFrame,
+    EndFrame,
+    Frame,
+    InterruptionFrame,
+    LLMFullResponseEndFrame,
+    LLMTextFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
+from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator
+
+
+class LLMTextProcessor(FrameProcessor):
+    """A processor for handling or manipulating LLM text frames before they are processed further.
+
+    This processor will convert LLMTextFrames into AggregatedTextFrames based on the configured
+    text aggregator. Using the customizable aggregator, it provides functionality to handle or
+    manipulate LLM text frames before they are sent to other components such as TTS services or
+    context aggregators. It can be used to pre-aggregate and categorize, modify, or filter direct
+    output tokens from the LLM.
+    """
+
+    def __init__(self, *, text_aggregator: Optional[BaseTextAggregator] = None, **kwargs):
+        """Initialize the LLM text processor.
+
+        Args:
+            text_aggregator: An optional text aggregator to use for processing LLM text frames. By
+                default, a SimpleTextAggregator aggregating by sentence will be used.
+            **kwargs: Additional arguments passed to parent class.
+
+        TODO: Allow transformations per aggregation type or all (and deprecate the TTS filters).
+        """
+        super().__init__(**kwargs)
+        self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process an LLMTextFrames using the aggregator to generate AggregatedTextFrames.
+
+        Args:
+            frame: The frame to process.
+            direction: The direction of frame flow in the pipeline.
+        """
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, InterruptionFrame):
+            await self._handle_interruption(frame)
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, LLMTextFrame):
+            await self._handle_llm_text(frame)
+        elif isinstance(frame, LLMFullResponseEndFrame):
+            await self._handle_llm_end(frame.skip_tts)
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, EndFrame):
+            await self._handle_llm_end()
+            await self.push_frame(frame, direction)
+        else:
+            await self.push_frame(frame, direction)
+
+    async def _handle_interruption(self, _):
+        """Handle interruptions by resetting the text aggregator."""
+        await self._text_aggregator.handle_interruption()
+
+    async def reset(self):
+        """Reset the internal state of the text processor and its aggregator."""
+        await self._text_aggregator.reset()
+
+    async def _handle_llm_text(self, in_frame: LLMTextFrame):
+        aggregation = await self._text_aggregator.aggregate(in_frame.text)
+        if aggregation:
+            out_frame = AggregatedTextFrame(
+                text=aggregation.text,
+                aggregated_by=aggregation.type,
+            )
+            out_frame.skip_tts = in_frame.skip_tts
+            await self.push_frame(out_frame)
+
+    async def _handle_llm_end(self, skip_tts: bool = False):
+        # Flush any remaining aggregated text at the end of the LLM response
+        aggregation = self._text_aggregator.text
+        await self._text_aggregator.reset()
+        text = aggregation.text.strip()
+        if text:
+            out_frame = AggregatedTextFrame(
+                text=text,
+                aggregated_by=aggregation.type,
+            )
+            out_frame.skip_tts = skip_tts
+            await self.push_frame(out_frame)

src/pipecat/processors/consumer_processor.py

@@ -83,4 +83,4 @@ class ConsumerProcessor(FrameProcessor):
         while True:
             frame = await self._queue.get()
             new_frame = await self._transformer(frame)
-            await self.push_frame(new_frame, self._direction)
+            await self.queue_frame(new_frame, self._direction)

src/pipecat/processors/frameworks/langchain.py

@@ -107,7 +107,9 @@ class LangchainProcessor(FrameProcessor):
                 {self._transcript_key: text},
                 config={"configurable": {"session_id": self._participant_id}},
             ):
-                await self.push_frame(TextFrame(self.__get_token_value(token)))
+                frame = TextFrame(self.__get_token_value(token))
+                frame.includes_inter_frame_spaces = True
+                await self.push_frame(frame)
         except GeneratorExit:
             logger.warning(f"{self} generator was closed prematurely")
         except Exception as e:

src/pipecat/processors/frameworks/rtvi.py

@@ -24,6 +24,7 @@ from typing import (
     Literal,
     Mapping,
     Optional,
+    Tuple,
     Union,
 )
 
@@ -32,6 +33,8 @@ from pydantic import BaseModel, Field, PrivateAttr, ValidationError
 
 from pipecat.audio.utils import calculate_audio_volume
 from pipecat.frames.frames import (
+    AggregatedTextFrame,
+    AggregationType,
     BotStartedSpeakingFrame,
     BotStoppedSpeakingFrame,
     CancelFrame,
@@ -704,6 +707,29 @@ class RTVITextMessageData(BaseModel):
     text: str
 
 
+class RTVIBotOutputMessageData(RTVITextMessageData):
+    """Data for bot output RTVI messages.
+
+    Extends RTVITextMessageData to include metadata about the output.
+    """
+
+    spoken: bool = False  # Indicates if the text has been spoken by TTS
+    aggregated_by: AggregationType | str
+    # Indicates what form the text is in (e.g., by word, sentence, etc.)
+
+
+class RTVIBotOutputMessage(BaseModel):
+    """Message containing bot output text.
+
+    An event meant to holistically represent what the bot is outputting,
+    along with metadata about the output and if it has been spoken.
+    """
+
+    label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
+    type: Literal["bot-output"] = "bot-output"
+    data: RTVIBotOutputMessageData
+
+
 class RTVIBotTranscriptionMessage(BaseModel):
     """Message containing bot transcription text.
 
@@ -896,6 +922,7 @@ class RTVIObserverParams:
         Parameter `errors_enabled` is deprecated. Error messages are always enabled.
 
     Parameters:
+        bot_output_enabled: Indicates if bot output messages should be sent.
         bot_llm_enabled: Indicates if the bot's LLM messages should be sent.
         bot_tts_enabled: Indicates if the bot's TTS messages should be sent.
         bot_speaking_enabled: Indicates if the bot's started/stopped speaking messages should be sent.
@@ -907,9 +934,17 @@ class RTVIObserverParams:
         metrics_enabled: Indicates if metrics messages should be sent.
         system_logs_enabled: Indicates if system logs should be sent.
         errors_enabled: [Deprecated] Indicates if errors messages should be sent.
+        skip_aggregator_types: List of aggregation types to skip sending as tts/output messages.
+          Note: if using this to avoid sending secure information, be sure to also disable
+                bot_llm_enabled to avoid leaking through LLM messages.
+        bot_output_transforms: A list of callables to transform text before just before sending it
+            to TTS. Each callable takes the aggregated text and its type, and returns the
+            transformed text. To register, provide a list of tuples of
+            (aggregation_type | '*', transform_function).
         audio_level_period_secs: How often audio levels should be sent if enabled.
     """
 
+    bot_output_enabled: bool = True
     bot_llm_enabled: bool = True
     bot_tts_enabled: bool = True
     bot_speaking_enabled: bool = True
@@ -921,6 +956,15 @@ class RTVIObserverParams:
     metrics_enabled: bool = True
     system_logs_enabled: bool = False
     errors_enabled: Optional[bool] = None
+    skip_aggregator_types: Optional[List[AggregationType | str]] = None
+    bot_output_transforms: Optional[
+        List[
+            Tuple[
+                AggregationType | str,
+                Callable[[str, AggregationType | str], Awaitable[str]],
+            ]
+        ]
+    ] = None
     audio_level_period_secs: float = 0.15
 
 
@@ -973,8 +1017,45 @@ class RTVIObserver(BaseObserver):
                     DeprecationWarning,
                 )
 
+        self._aggregation_transforms: List[
+            Tuple[AggregationType | str, Callable[[str, AggregationType | str], Awaitable[str]]]
+        ] = self._params.bot_output_transforms or []
+
+    def add_bot_output_transformer(
+        self,
+        transform_function: Callable[[str, AggregationType | str], Awaitable[str]],
+        aggregation_type: AggregationType | str = "*",
+    ):
+        """Transform text for a specific aggregation type before sending as Bot Output or TTS.
+
+        Args:
+            transform_function: The function to apply for transformation. This function should take
+                the text and aggregation type as input and return the transformed text.
+                Ex.: async def my_transform(text: str, aggregation_type: str) -> str:
+            aggregation_type: The type of aggregation to transform. This value defaults to "*" to
+                handle all text before sending to the client.
+        """
+        self._aggregation_transforms.append((aggregation_type, transform_function))
+
+    def remove_bot_output_transformer(
+        self,
+        transform_function: Callable[[str, AggregationType | str], Awaitable[str]],
+        aggregation_type: AggregationType | str = "*",
+    ):
+        """Remove a text transformer for a specific aggregation type.
+
+        Args:
+            transform_function: The function to remove.
+            aggregation_type: The type of aggregation to remove the transformer for.
+        """
+        self._aggregation_transforms = [
+            (agg_type, func)
+            for agg_type, func in self._aggregation_transforms
+            if not (agg_type == aggregation_type and func == transform_function)
+        ]
+
     async def _logger_sink(self, message):
-        """Logger sink so we cna send system logs to RTVI clients."""
+        """Logger sink so we can send system logs to RTVI clients."""
         message = RTVISystemLogMessage(data=RTVITextMessageData(text=message))
         await self.send_rtvi_message(message)
 
@@ -1048,12 +1129,15 @@ class RTVIObserver(BaseObserver):
             await self.send_rtvi_message(RTVIBotTTSStartedMessage())
         elif isinstance(frame, TTSStoppedFrame) and self._params.bot_tts_enabled:
             await self.send_rtvi_message(RTVIBotTTSStoppedMessage())
-        elif isinstance(frame, TTSTextFrame) and self._params.bot_tts_enabled:
-            if isinstance(src, BaseOutputTransport):
-                message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text))
-                await self.send_rtvi_message(message)
-            else:
+        elif isinstance(frame, AggregatedTextFrame) and (
+            self._params.bot_output_enabled or self._params.bot_tts_enabled
+        ):
+            if isinstance(frame, TTSTextFrame) and not isinstance(src, BaseOutputTransport):
+                # This check is to make sure we handle the frame when it has gone
+                # through the transport and has correct timing.
                 mark_as_seen = False
+            else:
+                await self._handle_aggregated_llm_text(frame)
         elif isinstance(frame, MetricsFrame) and self._params.metrics_enabled:
             await self._handle_metrics(frame)
         elif isinstance(frame, RTVIServerMessageFrame):
@@ -1084,15 +1168,6 @@ class RTVIObserver(BaseObserver):
         if mark_as_seen:
             self._frames_seen.add(frame.id)
 
-    async def _push_bot_transcription(self):
-        """Push accumulated bot transcription as a message."""
-        if len(self._bot_transcription) > 0:
-            message = RTVIBotTranscriptionMessage(
-                data=RTVITextMessageData(text=self._bot_transcription)
-            )
-            await self.send_rtvi_message(message)
-            self._bot_transcription = ""
-
     async def _handle_interruptions(self, frame: Frame):
         """Handle user speaking interruption frames."""
         message = None
@@ -1115,14 +1190,45 @@ class RTVIObserver(BaseObserver):
         if message:
             await self.send_rtvi_message(message)
 
+    async def _handle_aggregated_llm_text(self, frame: AggregatedTextFrame):
+        """Handle aggregated LLM text output frames."""
+        # Skip certain aggregator types if configured to do so.
+        if (
+            self._params.skip_aggregator_types
+            and frame.aggregated_by in self._params.skip_aggregator_types
+        ):
+            return
+
+        text = frame.text
+        type = frame.aggregated_by
+        for aggregation_type, transform in self._aggregation_transforms:
+            if aggregation_type == type or aggregation_type == "*":
+                text = await transform(text, type)
+
+        isTTS = isinstance(frame, TTSTextFrame)
+        if self._params.bot_output_enabled:
+            message = RTVIBotOutputMessage(
+                data=RTVIBotOutputMessageData(text=text, spoken=isTTS, aggregated_by=type)
+            )
+            await self.send_rtvi_message(message)
+
+        if isTTS and self._params.bot_tts_enabled:
+            tts_message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=text))
+            await self.send_rtvi_message(tts_message)
+
     async def _handle_llm_text_frame(self, frame: LLMTextFrame):
         """Handle LLM text output frames."""
         message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text))
         await self.send_rtvi_message(message)
 
+        # TODO (mrkb): Remove all this logic when we fully deprecate bot-transcription messages.
         self._bot_transcription += frame.text
-        if match_endofsentence(self._bot_transcription):
-            await self._push_bot_transcription()
+
+        if match_endofsentence(self._bot_transcription) and len(self._bot_transcription) > 0:
+            await self.send_rtvi_message(
+                RTVIBotTranscriptionMessage(data=RTVITextMessageData(text=self._bot_transcription))
+            )
+            self._bot_transcription = ""
 
     async def _handle_user_transcriptions(self, frame: Frame):
         """Handle user transcription frames."""
@@ -1248,7 +1354,7 @@ class RTVIProcessor(FrameProcessor):
         # Default to 0.3.0 which is the last version before actually having a
         # "client-version".
         self._client_version = [0, 3, 0]
-        self._skip_tts: bool = False  # Keep in sync with llm_service.py
+        self._llm_skip_tts: bool = False  # Keep in sync with llm_service.py's configuration.
 
         self._registered_actions: Dict[str, RTVIAction] = {}
         self._registered_services: Dict[str, RTVIService] = {}
@@ -1441,7 +1547,7 @@ class RTVIProcessor(FrameProcessor):
         elif isinstance(frame, RTVIActionFrame):
             await self._action_queue.put(frame)
         elif isinstance(frame, LLMConfigureOutputFrame):
-            self._skip_tts = frame.skip_tts
+            self._llm_skip_tts = frame.skip_tts
             await self.push_frame(frame, direction)
         # Other frames
         else:
@@ -1697,9 +1803,9 @@ class RTVIProcessor(FrameProcessor):
         opts = data.options if data.options is not None else RTVISendTextOptions()
         if opts.run_immediately:
             await self.interrupt_bot()
-        cur_skip_tts = self._skip_tts
+        cur_llm_skip_tts = self._llm_skip_tts
         should_skip_tts = not opts.audio_response
-        toggle_skip_tts = cur_skip_tts != should_skip_tts
+        toggle_skip_tts = cur_llm_skip_tts != should_skip_tts
         if toggle_skip_tts:
             output_frame = LLMConfigureOutputFrame(skip_tts=should_skip_tts)
             await self.push_frame(output_frame)
@@ -1709,7 +1815,7 @@ class RTVIProcessor(FrameProcessor):
         )
         await self.push_frame(text_frame)
         if toggle_skip_tts:
-            output_frame = LLMConfigureOutputFrame(skip_tts=cur_skip_tts)
+            output_frame = LLMConfigureOutputFrame(skip_tts=cur_llm_skip_tts)
             await self.push_frame(output_frame)
 
     async def _handle_update_context(self, data: RTVIAppendToContextData):

src/pipecat/processors/transcript_processor.py

@@ -26,7 +26,7 @@ from pipecat.frames.frames import (
     TTSTextFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.utils.string import concatenate_aggregated_text
+from pipecat.utils.string import TextPartForConcatenation, concatenate_aggregated_text
 from pipecat.utils.time import time_now_iso8601
 
 
@@ -98,7 +98,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
             **kwargs: Additional arguments passed to parent class.
         """
         super().__init__(**kwargs)
-        self._current_text_parts: List[str] = []
+        self._current_text_parts: List[TextPartForConcatenation] = []
         self._aggregation_start_time: Optional[str] = None
 
     async def _emit_aggregated_text(self):
@@ -185,7 +185,11 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
             if not self._aggregation_start_time:
                 self._aggregation_start_time = time_now_iso8601()
 
-            self._current_text_parts.append(frame.text)
+            self._current_text_parts.append(
+                TextPartForConcatenation(
+                    frame.text, includes_inter_part_spaces=frame.includes_inter_frame_spaces
+                )
+            )
 
             # Push frame.
             await self.push_frame(frame, direction)

src/pipecat/runner/run.py

@@ -264,7 +264,10 @@ def _setup_webrtc_routes(
         # Prepare runner arguments with the callback to run your bot
         async def webrtc_connection_callback(connection):
             bot_module = _get_bot_module()
-            runner_args = SmallWebRTCRunnerArguments(webrtc_connection=connection)
+
+            runner_args = SmallWebRTCRunnerArguments(
+                webrtc_connection=connection, body=request.request_data
+            )
             background_tasks.add_task(bot_module.bot, runner_args)
 
         # Delegate handling to SmallWebRTCRequestHandler
@@ -326,7 +329,8 @@ def _setup_webrtc_routes(
                         type=request_data["type"],
                         pc_id=request_data.get("pc_id"),
                         restart_pc=request_data.get("restart_pc"),
-                        request_data=request_data,
+                        request_data=request_data.get("request_data")
+                        or request_data.get("requestData"),
                     )
                     return await offer(webrtc_request, background_tasks)
                 elif request.method == HTTPMethod.PATCH.value:

src/pipecat/runner/utils.py

@@ -281,6 +281,14 @@ async def maybe_capture_participant_camera(
     except ImportError:
         pass
 
+    try:
+        from pipecat.transports.smallwebrtc.transport import SmallWebRTCTransport
+
+        if isinstance(transport, SmallWebRTCTransport):
+            await transport.capture_participant_video(video_source="camera")
+    except ImportError:
+        pass
+
 
 async def maybe_capture_participant_screen(
     transport: BaseTransport, client: Any, framerate: int = 0
@@ -303,6 +311,14 @@ async def maybe_capture_participant_screen(
     except ImportError:
         pass
 
+    try:
+        from pipecat.transports.smallwebrtc.transport import SmallWebRTCTransport
+
+        if isinstance(transport, SmallWebRTCTransport):
+            await transport.capture_participant_video(video_source="screenVideo")
+    except ImportError:
+        pass
+
 
 def _smallwebrtc_sdp_cleanup_ice_candidates(text: str, pattern: str) -> str:
     """Clean up ICE candidates in SDP text for SmallWebRTC.

src/pipecat/serializers/livekit.py

@@ -1,98 +0,0 @@
-#
-# Copyright (c) 2024–2025, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-"""LiveKit frame serializer for Pipecat."""
-
-import ctypes
-import pickle
-
-from loguru import logger
-
-from pipecat.frames.frames import Frame, InputAudioRawFrame, OutputAudioRawFrame
-from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
-
-try:
-    from livekit.rtc import AudioFrame
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error("In order to use LiveKit, you need to `pip install pipecat-ai[livekit]`.")
-    raise Exception(f"Missing module: {e}")
-
-
-class LivekitFrameSerializer(FrameSerializer):
-    """Serializer for converting between Pipecat frames and LiveKit audio frames.
-
-    .. deprecated:: 0.0.90
-
-        This class is deprecated and will be removed in a future version.
-        Please use LiveKitTransport instead, which handles audio streaming
-        and frame conversion natively.
-
-    This serializer handles the conversion of Pipecat's OutputAudioRawFrame objects
-    to LiveKit AudioFrame objects for transmission, and the reverse conversion
-    for received audio data.
-    """
-
-    def __init__(self):
-        """Initialize the LiveKit frame serializer."""
-        super().__init__()
-        import warnings
-
-        with warnings.catch_warnings():
-            warnings.simplefilter("always")
-            warnings.warn(
-                "LivekitFrameSerializer is deprecated and will be removed in a future version. "
-                "Please use LiveKitTransport instead, which handles audio streaming natively.",
-                DeprecationWarning,
-                stacklevel=2,
-            )
-
-    @property
-    def type(self) -> FrameSerializerType:
-        """Get the serializer type.
-
-        Returns:
-            The serializer type indicating binary serialization.
-        """
-        return FrameSerializerType.BINARY
-
-    async def serialize(self, frame: Frame) -> str | bytes | None:
-        """Serialize a Pipecat frame to LiveKit AudioFrame format.
-
-        Args:
-            frame: The Pipecat frame to serialize. Only OutputAudioRawFrame
-                  instances are supported.
-
-        Returns:
-            Pickled LiveKit AudioFrame bytes if frame is OutputAudioRawFrame,
-            None otherwise.
-        """
-        if not isinstance(frame, OutputAudioRawFrame):
-            return None
-        audio_frame = AudioFrame(
-            data=frame.audio,
-            sample_rate=frame.sample_rate,
-            num_channels=frame.num_channels,
-            samples_per_channel=len(frame.audio) // ctypes.sizeof(ctypes.c_int16),
-        )
-        return pickle.dumps(audio_frame)
-
-    async def deserialize(self, data: str | bytes) -> Frame | None:
-        """Deserialize LiveKit AudioFrame data to a Pipecat frame.
-
-        Args:
-            data: Pickled data containing a LiveKit AudioFrame.
-
-        Returns:
-            InputAudioRawFrame containing the deserialized audio data,
-            or None if deserialization fails.
-        """
-        audio_frame: AudioFrame = pickle.loads(data)["frame"]
-        return InputAudioRawFrame(
-            audio=bytes(audio_frame.data),
-            sample_rate=audio_frame.sample_rate,
-            num_channels=audio_frame.num_channels,
-        )

src/pipecat/services/assemblyai/stt.py

@@ -21,6 +21,7 @@ from pipecat import __version__ as pipecat_version
 from pipecat.frames.frames import (
     CancelFrame,
     EndFrame,
+    ErrorFrame,
     Frame,
     InterimTranscriptionFrame,
     StartFrame,
@@ -205,8 +206,9 @@ class AssemblyAISTTService(STTService):
 
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"Failed to connect to AssemblyAI: {e}")
+            logger.error(f"{self} exception: {e}")
             self._connected = False
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             raise
 
     async def _disconnect(self):
@@ -231,7 +233,8 @@ class AssemblyAISTTService(STTService):
                     logger.warning("Timed out waiting for termination message from server")
 
             except Exception as e:
-                logger.warning(f"Error during termination handshake: {e}")
+                logger.error(f"{self} exception: {e}")
+                await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
             if self._receive_task:
                 await self.cancel_task(self._receive_task)
@@ -239,7 +242,8 @@ class AssemblyAISTTService(STTService):
             await self._websocket.close()
 
         except Exception as e:
-            logger.error(f"Error during disconnect: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
         finally:
             self._websocket = None
@@ -258,11 +262,13 @@ class AssemblyAISTTService(STTService):
                 except websockets.exceptions.ConnectionClosedOK:
                     break
                 except Exception as e:
-                    logger.error(f"Error processing WebSocket message: {e}")
+                    logger.error(f"{self} exception: {e}")
+                    await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
                     break
 
         except Exception as e:
-            logger.error(f"Fatal error in receive handler: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
     def _parse_message(self, message: Dict[str, Any]) -> BaseMessage:
         """Parse a raw message into the appropriate message type."""
@@ -291,7 +297,8 @@ class AssemblyAISTTService(STTService):
             elif isinstance(parsed_message, TerminationMessage):
                 await self._handle_termination(parsed_message)
         except Exception as e:
-            logger.error(f"Error handling message: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
     async def _handle_termination(self, message: TerminationMessage):
         """Handle termination message."""

src/pipecat/services/asyncai/tts.py

@@ -228,7 +228,8 @@ class AsyncAITTSService(InterruptibleTTSService):
 
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"{self} initialization error: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
 
@@ -240,7 +241,8 @@ class AsyncAITTSService(InterruptibleTTSService):
                 logger.debug("Disconnecting from Async")
                 await self._websocket.close()
         except Exception as e:
-            logger.error(f"{self} error closing websocket: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._websocket = None
             self._started = False
@@ -288,7 +290,7 @@ class AsyncAITTSService(InterruptibleTTSService):
                 logger.error(f"{self} error: {msg}")
                 await self.push_frame(TTSStoppedFrame())
                 await self.stop_all_metrics()
-                await self.push_error(ErrorFrame(f"{self} error: {msg['message']}"))
+                await self.push_error(ErrorFrame(error=f"{self} error: {msg['message']}"))
             else:
                 logger.error(f"{self} error, unknown message type: {msg}")
 
@@ -333,7 +335,8 @@ class AsyncAITTSService(InterruptibleTTSService):
                 await self._get_websocket().send(msg)
                 await self.start_tts_usage_metrics(text)
             except Exception as e:
-                logger.error(f"{self} error sending message: {e}")
+                logger.error(f"{self} exception: {e}")
+                yield ErrorFrame(error=f"{self} error: {e}")
                 yield TTSStoppedFrame()
                 await self._disconnect()
                 await self._connect()
@@ -341,6 +344,7 @@ class AsyncAITTSService(InterruptibleTTSService):
             yield None
         except Exception as e:
             logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
 
 class AsyncAIHttpTTSService(TTSService):
@@ -474,7 +478,7 @@ class AsyncAIHttpTTSService(TTSService):
                 if response.status != 200:
                     error_text = await response.text()
                     logger.error(f"Async API error: {error_text}")
-                    await self.push_error(ErrorFrame(f"Async API error: {error_text}"))
+                    await self.push_error(ErrorFrame(error=f"Async API error: {error_text}"))
                     raise Exception(f"Async API returned status {response.status}: {error_text}")
 
                 audio_data = await response.read()
@@ -491,7 +495,7 @@ class AsyncAIHttpTTSService(TTSService):
 
         except Exception as e:
             logger.error(f"{self} exception: {e}")
-            await self.push_error(ErrorFrame(f"Error generating TTS: {e}"))
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             await self.stop_ttfb_metrics()
             yield TTSStoppedFrame()

src/pipecat/services/aws/__init__.py

@@ -8,8 +8,10 @@ import sys
 
 from pipecat.services import DeprecatedModuleProxy
 
+from .agent_core import *
 from .llm import *
 from .nova_sonic import *
+from .sagemaker import *
 from .stt import *
 from .tts import *
 

src/pipecat/services/aws/agent_core.py

@@ -0,0 +1,258 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""AWS AgentCore Processor Module.
+
+This module defines the AWSAgentCoreProcessor, which invokes agents hosted on
+Amazon Bedrock AgentCore Runtime and streams their responses as LLMTextFrames.
+"""
+
+import asyncio
+import json
+import os
+from typing import Callable, Optional
+
+import aioboto3
+from loguru import logger
+
+from pipecat.frames.frames import (
+    Frame,
+    LLMContextFrame,
+    LLMFullResponseEndFrame,
+    LLMFullResponseStartFrame,
+    LLMTextFrame,
+)
+from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
+from pipecat.processors.aggregators.openai_llm_context import (
+    OpenAILLMContext,
+    OpenAILLMContextFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+
+
+def default_context_to_payload_transformer(
+    context: LLMContext | OpenAILLMContext,
+) -> Optional[str]:
+    """Default transformer to create AgentCore payload from LLM context.
+
+    Extracts the latest user or system message text and wraps it in {"prompt": "<text>"}.
+
+    Args:
+        context: The LLM context containing conversation messages.
+
+    Returns:
+        A JSON string payload for AgentCore, or None if no valid message found.
+    """
+    messages = context.messages
+
+    if not messages:
+        return None
+
+    last_message = messages[-1]
+    if isinstance(last_message, LLMSpecificMessage) or last_message.get("role") not in (
+        "user",
+        "system",
+    ):
+        return None
+
+    content = last_message.get("content")
+    if not content:
+        return None
+
+    if isinstance(content, str):
+        prompt = content
+    elif isinstance(content, list):
+        prompt = " ".join([part.get("text", "") for part in content])
+    else:
+        return None
+
+    return json.dumps({"prompt": prompt})
+
+
+def default_response_to_output_transformer(response_line: str) -> Optional[str]:
+    """Default transformer to extract output text from AgentCore response.
+
+    Expects responses with {"response": "<text>"} format.
+
+    Args:
+        response_line: The raw response line from AgentCore (without "data: " prefix).
+
+    Returns:
+        The extracted output text, or None if no text found.
+    """
+    response_json = json.loads(response_line)
+    return response_json.get("response")
+
+
+class AWSAgentCoreProcessor(FrameProcessor):
+    """Processor that runs an Amazon Bedrock AgentCore agent.
+
+    Input:
+        - LLMContextFrame: Supplies a context used to invoke the agent.
+
+    Output:
+        - LLMTextFrame: The agent's text response(s).
+          A single agent invocation may result in multiple text frames.
+
+    This processor transforms the input context to a payload for the AgentCore
+    agent, and transforms the agent's response(s) into output text frame(s). Both
+    mappings are configurable via transformers. Below is the default behavior.
+
+    Input transformer (context_to_payload_transformer):
+        - Grabs the latest user or system message (if it's the latest message)
+        - Extracts its text content
+        - Constructs a payload that looks like {"prompt": "<text>"}
+
+    Output transformer (response_to_output_transformer):
+        - Expects responses that look like {"response": "<text>"}
+        - Extracts the text for use in the LLMTextFrame(s)
+    """
+
+    def __init__(
+        self,
+        agentArn: str,
+        aws_access_key: Optional[str] = None,
+        aws_secret_key: Optional[str] = None,
+        aws_session_token: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        context_to_payload_transformer: Optional[
+            Callable[[LLMContext | OpenAILLMContext], Optional[str]]
+        ] = None,
+        response_to_output_transformer: Optional[Callable[[str], Optional[str]]] = None,
+        **kwargs,
+    ):
+        """Initialize the AWS AgentCore processor.
+
+        Args:
+            agentArn: The Amazon Web Services Resource Name (ARN) of the agent.
+            aws_access_key: AWS access key ID. If None, uses default credentials.
+            aws_secret_key: AWS secret access key. If None, uses default credentials.
+            aws_session_token: AWS session token for temporary credentials.
+            aws_region: AWS region.
+            context_to_payload_transformer: Optional callable to transform
+                LLMContext into AgentCore payload string. If None, uses
+                default_context_to_payload_transformer.
+            response_to_output_transformer: Optional callable to extract output text
+                from AgentCore response. If None, uses
+                default_response_to_output_transformer.
+            **kwargs: Additional arguments passed to parent FrameProcessor.
+        """
+        super().__init__(**kwargs)
+
+        self._agentArn = agentArn
+        self._aws_session = aioboto3.Session()
+
+        # Store AWS session parameters for creating client in async context
+        self._aws_params = {
+            "aws_access_key_id": aws_access_key or os.getenv("AWS_ACCESS_KEY_ID"),
+            "aws_secret_access_key": aws_secret_key or os.getenv("AWS_SECRET_ACCESS_KEY"),
+            "aws_session_token": aws_session_token or os.getenv("AWS_SESSION_TOKEN"),
+            "region_name": aws_region or os.getenv("AWS_REGION", "us-east-1"),
+        }
+
+        # Set transformers with defaults
+        self._context_to_payload_transformer = (
+            context_to_payload_transformer or default_context_to_payload_transformer
+        )
+        self._response_to_output_transformer = (
+            response_to_output_transformer or default_response_to_output_transformer
+        )
+
+        # State for managing output response bookends
+        self._output_response_open = False
+        self._last_text_frame_time: Optional[float] = None
+        self._close_task: Optional[asyncio.Task] = None
+        self._output_response_timeout = 1.0  # seconds
+
+    async def _close_output_response_after_timeout(self):
+        """Close the output response after timeout if no new text frames arrive."""
+        await asyncio.sleep(self._output_response_timeout)
+        if self._output_response_open:
+            self._output_response_open = False
+            await self.push_frame(LLMFullResponseEndFrame())
+
+    async def _push_text_frame(self, text: str):
+        """Push a text frame, managing output response bookends."""
+        # Cancel any pending close task
+        if self._close_task and not self._close_task.done():
+            await self.cancel_task(self._close_task)
+
+        # Open output response if needed
+        if not self._output_response_open:
+            await self.push_frame(LLMFullResponseStartFrame())
+            self._output_response_open = True
+
+        # Push the text frame
+        await self.push_frame(LLMTextFrame(text))
+        self._last_text_frame_time = asyncio.get_event_loop().time()
+
+        # Schedule closing the output response after timeout
+        self._close_task = self.create_task(self._close_output_response_after_timeout())
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames and handle LLM message frames.
+
+        Args:
+            frame: The incoming frame to process.
+            direction: The direction of frame flow in the pipeline.
+        """
+        await super().process_frame(frame, direction)
+        if isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)):
+            # Create payload to invoke AgentCore agent
+            payload = self._context_to_payload_transformer(frame.context)
+
+            if not payload:
+                return
+
+            async with self._aws_session.client("bedrock-agentcore", **self._aws_params) as client:
+                # Invoke the AgentCore agent
+                response = await client.invoke_agent_runtime(
+                    agentRuntimeArn=self._agentArn, payload=payload.encode()
+                )
+
+                # Determine if this is a streamed multi-part response, which
+                # will affect our parsing
+                is_multi_part_response = "text/event-stream" in response.get("contentType", "")
+
+                # Handle each response part (there may be one, for single
+                # responses, or multiple, for streamed multi-part responses)
+                async for part in response.get("response", []):
+                    part_string = part.decode("utf-8")
+
+                    # In streamed multi-part responses, each part might have
+                    # one or more lines, each of which starts with "data: ".
+                    # Treat each line as a response.
+                    if is_multi_part_response:
+                        for line in part_string.split("\n"):
+                            # Get response text from this line
+                            if not line:
+                                continue
+                            if not line.startswith("data: "):
+                                logger.warning(f"Expected line to start with 'data: ', got: {line}")
+                                continue
+                            line = line[6:]  # omit "data: "
+
+                            # Transform response line to output text
+                            text = self._response_to_output_transformer(line)
+                            if text:
+                                await self._push_text_frame(text)
+
+                    # In single-part responses, the whole part is one response
+                    # and there's no "data: " prefix
+                    else:
+                        # Transform response part string to output text
+                        text = self._response_to_output_transformer(part_string)
+                        if text:
+                            await self._push_text_frame(text)
+
+                # Final close if output response is still open after all parts processed
+                if self._output_response_open:
+                    if self._close_task and not self._close_task.done():
+                        await self.cancel_task(self._close_task)
+                    self._output_response_open = False
+                    await self.push_frame(LLMFullResponseEndFrame())
+        else:
+            await self.push_frame(frame, direction)

src/pipecat/services/aws/nova_sonic/llm.py

@@ -27,6 +27,7 @@ from pydantic import BaseModel, Field
 from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter, Role
 from pipecat.frames.frames import (
+    AggregationType,
     BotStoppedSpeakingFrame,
     CancelFrame,
     EndFrame,
@@ -1027,7 +1028,9 @@ class AWSNovaSonicLLMService(LLMService):
         logger.debug(f"Assistant response text added: {text}")
 
         # Report the text of the assistant response.
-        await self.push_frame(TTSTextFrame(text))
+        frame = TTSTextFrame(text, aggregated_by=AggregationType.SENTENCE)
+        frame.includes_inter_frame_spaces = True
+        await self.push_frame(frame)
 
         # HACK: here we're also buffering the assistant text ourselves as a
         # backup rather than relying solely on the assistant context aggregator
@@ -1060,7 +1063,11 @@ class AWSNovaSonicLLMService(LLMService):
                 # TTSTextFrame would be ignored otherwise (the interruption frame
                 # would have cleared the assistant aggregator state).
                 await self.push_frame(LLMFullResponseStartFrame())
-                await self.push_frame(TTSTextFrame(self._assistant_text_buffer))
+                frame = TTSTextFrame(
+                    self._assistant_text_buffer, aggregated_by=AggregationType.SENTENCE
+                )
+                frame.includes_inter_frame_spaces = True
+                await self.push_frame(frame)
             self._may_need_repush_assistant_text = False
 
         # Report the end of the assistant response.

src/pipecat/services/aws/sagemaker/bidi_client.py

@@ -0,0 +1,283 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""AWS SageMaker bidirectional streaming client.
+
+This module provides a client for streaming bidirectional communication with
+SageMaker endpoints using the HTTP/2 protocol. Supports sending audio, text,
+and JSON data to SageMaker model endpoints and receiving streaming responses.
+"""
+
+import os
+from typing import Optional
+
+from loguru import logger
+
+try:
+    from aws_sdk_sagemaker_runtime_http2.client import SageMakerRuntimeHTTP2Client
+    from aws_sdk_sagemaker_runtime_http2.config import Config, HTTPAuthSchemeResolver
+    from aws_sdk_sagemaker_runtime_http2.models import (
+        InvokeEndpointWithBidirectionalStreamInput,
+        RequestPayloadPart,
+        RequestStreamEventPayloadPart,
+        ResponseStreamEvent,
+    )
+    from smithy_aws_core.auth.sigv4 import SigV4AuthScheme
+    from smithy_aws_core.identity import EnvironmentCredentialsResolver
+    from smithy_core.aio.eventstream import DuplexEventStream
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use SageMaker BiDi client, you need to `pip install pipecat-ai[sagemaker]`."
+    )
+    raise Exception(f"Missing module: {e}")
+
+
+class SageMakerBidiClient:
+    """Client for bidirectional streaming with AWS SageMaker endpoints.
+
+    Handles low-level HTTP/2 bidirectional streaming protocol for communicating
+    with SageMaker model endpoints. Provides methods for sending various data
+    types (audio, text, JSON) and receiving streaming responses.
+
+    This client uses AWS SigV4 authentication and supports credential resolution
+    from environment variables, AWS CLI configuration, and instance metadata.
+
+    Example::
+
+        client = SageMakerBidiClient(
+            endpoint_name="my-deepgram-endpoint",
+            region="us-east-2",
+            model_invocation_path="v1/listen",
+            model_query_string="model=nova-3&language=en"
+        )
+        await client.start_session()
+        await client.send_audio_chunk(audio_bytes)
+        response = await client.receive_response()
+        await client.close_session()
+    """
+
+    def __init__(
+        self,
+        endpoint_name: str,
+        region: str,
+        model_invocation_path: str = "",
+        model_query_string: str = "",
+    ):
+        """Initialize the SageMaker BiDi client.
+
+        Args:
+            endpoint_name: Name of the SageMaker endpoint to connect to.
+            region: AWS region where the endpoint is deployed.
+            model_invocation_path: API path for the model invocation (e.g., "v1/listen").
+            model_query_string: Query string parameters for the model (e.g., "model=nova-3").
+        """
+        self.endpoint_name = endpoint_name
+        self.region = region
+        self.model_invocation_path = model_invocation_path
+        self.model_query_string = model_query_string
+        self.bidi_endpoint = f"https://runtime.sagemaker.{region}.amazonaws.com:8443"
+        self._client: Optional[SageMakerRuntimeHTTP2Client] = None
+        self._stream: Optional[
+            DuplexEventStream[RequestStreamEventPayloadPart, ResponseStreamEvent, any]
+        ] = None
+        self._output_stream = None
+        self._is_active = False
+
+    def _initialize_client(self):
+        """Initialize the SageMaker Runtime HTTP2 client with AWS credentials.
+
+        Creates and configures the SageMaker Runtime HTTP2 client with SigV4
+        authentication. Attempts to resolve AWS credentials from environment
+        variables, AWS CLI configuration, or instance metadata.
+        """
+        logger.debug(f"Initializing SageMaker BiDi client for region: {self.region}")
+        logger.debug(f"Using endpoint URI: {self.bidi_endpoint}")
+
+        # Check for AWS credentials
+        has_env_creds = bool(os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"))
+
+        if not has_env_creds:
+            logger.warning(
+                "AWS credentials not found in environment variables. "
+                "Attempting to use EnvironmentCredentialsResolver which will check "
+                "AWS CLI configuration and instance metadata."
+            )
+
+        config = Config(
+            endpoint_uri=self.bidi_endpoint,
+            region=self.region,
+            aws_credentials_identity_resolver=EnvironmentCredentialsResolver(),
+            auth_scheme_resolver=HTTPAuthSchemeResolver(),
+            auth_schemes={"aws.auth#sigv4": SigV4AuthScheme(service="sagemaker")},
+        )
+        self._client = SageMakerRuntimeHTTP2Client(config=config)
+
+    async def start_session(self):
+        """Start a bidirectional streaming session with the SageMaker endpoint.
+
+        Initializes the client if needed, creates the bidirectional stream, and
+        establishes the connection to the SageMaker endpoint. Must be called
+        before sending or receiving data.
+
+        Returns:
+            The output stream for receiving responses.
+
+        Raises:
+            RuntimeError: If client initialization or connection fails.
+        """
+        if not self._client:
+            self._initialize_client()
+
+        logger.debug(f"Starting BiDi session with endpoint: {self.endpoint_name}")
+        logger.debug(f"Model invocation path: {self.model_invocation_path}")
+        logger.debug(f"Model query string: {self.model_query_string}")
+
+        # Create the bidirectional stream
+        stream_input = InvokeEndpointWithBidirectionalStreamInput(
+            endpoint_name=self.endpoint_name,
+            model_invocation_path=self.model_invocation_path,
+            model_query_string=self.model_query_string,
+        )
+
+        try:
+            self._stream = await self._client.invoke_endpoint_with_bidirectional_stream(
+                stream_input
+            )
+            self._is_active = True
+
+            # Get output stream
+            output = await self._stream.await_output()
+            self._output_stream = output[1]
+
+            logger.debug("BiDi session started successfully")
+            return self._output_stream
+
+        except Exception as e:
+            logger.error(f"Failed to start BiDi session: {e}")
+            self._is_active = False
+            raise RuntimeError(f"Failed to start SageMaker BiDi session: {e}")
+
+    async def send_data(self, data_bytes: bytes, data_type: Optional[str] = None):
+        """Send a chunk of data to the stream.
+
+        Generic method for sending any type of data to the SageMaker endpoint.
+        Use the convenience methods (send_audio_chunk, send_text, send_json)
+        for common data types.
+
+        Args:
+            data_bytes: Raw bytes to send.
+            data_type: Optional data type header. Common values are "BINARY" for
+                audio/binary data and "UTF8" for text/JSON data.
+
+        Raises:
+            RuntimeError: If session is not active or send fails.
+        """
+        if not self._is_active or not self._stream:
+            raise RuntimeError("BiDi session not active")
+
+        try:
+            payload = RequestPayloadPart(bytes_=data_bytes, data_type=data_type)
+            event = RequestStreamEventPayloadPart(value=payload)
+            await self._stream.input_stream.send(event)
+        except Exception as e:
+            logger.error(f"Failed to send data: {e}")
+            raise
+
+    async def send_audio_chunk(self, audio_bytes: bytes):
+        """Send a chunk of audio data to the stream.
+
+        Convenience method for sending audio data. Automatically sets the data
+        type to "BINARY".
+
+        Args:
+            audio_bytes: Raw audio bytes to send (e.g., PCM audio data).
+
+        Raises:
+            RuntimeError: If session is not active or send fails.
+        """
+        await self.send_data(audio_bytes, data_type="BINARY")
+
+    async def send_text(self, text: str):
+        """Send text data to the stream.
+
+        Convenience method for sending text data. Automatically encodes the text
+        as UTF-8 and sets the data type to "UTF8".
+
+        Args:
+            text: Text string to send.
+
+        Raises:
+            RuntimeError: If session is not active or send fails.
+        """
+        await self.send_data(text.encode("utf-8"), data_type="UTF8")
+
+    async def send_json(self, data: dict):
+        """Send JSON data to the stream.
+
+        Convenience method for sending JSON-encoded messages. Useful for control
+        messages like KeepAlive or CloseStream. Automatically serializes the
+        dictionary to JSON, encodes as UTF-8, and sets the data type to "UTF8".
+
+        Args:
+            data: Dictionary to send as JSON (e.g., {"type": "KeepAlive"}).
+
+        Raises:
+            RuntimeError: If session is not active or send fails.
+        """
+        import json
+
+        await self.send_data(json.dumps(data).encode("utf-8"), data_type="UTF8")
+
+    async def receive_response(self) -> Optional[ResponseStreamEvent]:
+        """Receive a response from the stream.
+
+        Blocks until a response is available from the SageMaker endpoint. Returns
+        None when the stream is closed.
+
+        Returns:
+            The response event containing payload data, or None if stream is closed.
+
+        Raises:
+            RuntimeError: If session is not active.
+        """
+        if not self._is_active or not self._output_stream:
+            raise RuntimeError("BiDi session not active")
+
+        try:
+            result = await self._output_stream.receive()
+            return result
+        except Exception as e:
+            logger.error(f"Failed to receive response: {e}")
+            raise
+
+    async def close_session(self):
+        """Close the bidirectional streaming session.
+
+        Gracefully closes the input stream and marks the session as inactive.
+        Safe to call multiple times.
+        """
+        if not self._is_active:
+            return
+
+        logger.debug("Closing BiDi session...")
+        self._is_active = False
+
+        try:
+            if self._stream:
+                await self._stream.input_stream.close()
+            logger.debug("BiDi session closed successfully")
+        except Exception as e:
+            logger.warning(f"Error closing BiDi session: {e}")
+
+    @property
+    def is_active(self) -> bool:
+        """Check if the session is currently active.
+
+        Returns:
+            True if session is active, False otherwise.
+        """
+        return self._is_active

src/pipecat/services/aws/stt.py

@@ -140,7 +140,8 @@ class AWSTranscribeSTTService(STTService):
                     return
                 logger.warning("WebSocket connection not established after connect")
             except Exception as e:
-                logger.error(f"Failed to connect (attempt {retry_count + 1}/{max_retries}): {e}")
+                logger.error(f"{self} exception: {e}")
+                await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
                 retry_count += 1
                 if retry_count < max_retries:
                     await asyncio.sleep(1)  # Wait before retrying
@@ -181,8 +182,8 @@ class AWSTranscribeSTTService(STTService):
                 try:
                     await self._connect()
                 except Exception as e:
-                    logger.error(f"Failed to reconnect: {e}")
-                    yield ErrorFrame("Failed to reconnect to AWS Transcribe", fatal=False)
+                    logger.error(f"{self} exception: {e}")
+                    yield ErrorFrame(error=f"{self} error: {e}")
                     return
 
             # Format the audio data according to AWS event stream format
@@ -199,13 +200,13 @@ class AWSTranscribeSTTService(STTService):
                 await self._disconnect()
                 # Don't yield error here - we'll retry on next frame
             except Exception as e:
-                logger.error(f"Error sending audio: {e}")
-                yield ErrorFrame(f"AWS Transcribe error: {str(e)}", fatal=False)
+                logger.error(f"{self} exception: {e}")
+                yield ErrorFrame(error=f"{self} error: {e}")
                 await self._disconnect()
 
         except Exception as e:
-            logger.error(f"Error in run_stt: {e}")
-            yield ErrorFrame(f"AWS Transcribe error: {str(e)}", fatal=False)
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
             await self._disconnect()
 
     async def _connect(self):
@@ -288,7 +289,8 @@ class AWSTranscribeSTTService(STTService):
 
                 await self._call_event_handler("on_connected")
             except Exception as e:
-                logger.error(f"{self} Failed to connect to AWS Transcribe: {e}")
+                logger.error(f"{self} exception: {e}")
+                await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
                 await self._disconnect()
                 raise
 
@@ -308,7 +310,8 @@ class AWSTranscribeSTTService(STTService):
                 await self._ws_client.send(json.dumps(end_stream))
             await self._ws_client.close()
         except Exception as e:
-            logger.warning(f"{self} Error closing WebSocket connection: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._ws_client = None
             await self._call_event_handler("on_disconnected")
@@ -527,9 +530,7 @@ class AWSTranscribeSTTService(STTService):
                 elif headers.get(":message-type") == "exception":
                     error_msg = payload.get("Message", "Unknown error")
                     logger.error(f"{self} Exception from AWS: {error_msg}")
-                    await self.push_frame(
-                        ErrorFrame(f"AWS Transcribe error: {error_msg}", fatal=False)
-                    )
+                    await self.push_frame(ErrorFrame(f"AWS Transcribe error: {error_msg}"))
                 else:
                     logger.debug(f"{self} Other message type received: {headers}")
                     logger.debug(f"{self} Payload: {payload}")
@@ -537,5 +538,6 @@ class AWSTranscribeSTTService(STTService):
                 logger.error(f"{self} WebSocket connection closed in receive loop: {e}")
                 break
             except Exception as e:
-                logger.error(f"{self} Unexpected error in receive loop: {e}")
+                logger.error(f"{self} exception: {e}")
+                await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
                 break

src/pipecat/services/azure/stt.py

@@ -18,6 +18,7 @@ from loguru import logger
 from pipecat.frames.frames import (
     CancelFrame,
     EndFrame,
+    ErrorFrame,
     Frame,
     InterimTranscriptionFrame,
     StartFrame,
@@ -111,13 +112,17 @@ class AzureSTTService(STTService):
             audio: Raw audio bytes to process.
 
         Yields:
-            None - actual transcription frames are pushed via callbacks.
+            Frame: Either None for successful processing or ErrorFrame on failure.
         """
-        await self.start_processing_metrics()
-        await self.start_ttfb_metrics()
-        if self._audio_stream:
-            self._audio_stream.write(audio)
-        yield None
+        try:
+            await self.start_processing_metrics()
+            await self.start_ttfb_metrics()
+            if self._audio_stream:
+                self._audio_stream.write(audio)
+            yield None
+        except Exception as e:
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
     async def start(self, frame: StartFrame):
         """Start the speech recognition service.
@@ -133,17 +138,21 @@ class AzureSTTService(STTService):
         if self._audio_stream:
             return
 
-        stream_format = AudioStreamFormat(samples_per_second=self.sample_rate, channels=1)
-        self._audio_stream = PushAudioInputStream(stream_format)
+        try:
+            stream_format = AudioStreamFormat(samples_per_second=self.sample_rate, channels=1)
+            self._audio_stream = PushAudioInputStream(stream_format)
 
-        audio_config = AudioConfig(stream=self._audio_stream)
+            audio_config = AudioConfig(stream=self._audio_stream)
 
-        self._speech_recognizer = SpeechRecognizer(
-            speech_config=self._speech_config, audio_config=audio_config
-        )
-        self._speech_recognizer.recognizing.connect(self._on_handle_recognizing)
-        self._speech_recognizer.recognized.connect(self._on_handle_recognized)
-        self._speech_recognizer.start_continuous_recognition_async()
+            self._speech_recognizer = SpeechRecognizer(
+                speech_config=self._speech_config, audio_config=audio_config
+            )
+            self._speech_recognizer.recognizing.connect(self._on_handle_recognizing)
+            self._speech_recognizer.recognized.connect(self._on_handle_recognized)
+            self._speech_recognizer.start_continuous_recognition_async()
+        except Exception as e:
+            logger.error(f"{self} exception during initialization: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
     async def stop(self, frame: EndFrame):
         """Stop the speech recognition service.

src/pipecat/services/azure/tts.py

@@ -328,7 +328,7 @@ class AzureTTSService(AzureBaseTTSService):
             if self._speech_synthesizer is None:
                 error_msg = "Speech synthesizer not initialized."
                 logger.error(error_msg)
-                yield ErrorFrame(error_msg)
+                yield ErrorFrame(error=error_msg)
                 return
 
             try:
@@ -355,13 +355,15 @@ class AzureTTSService(AzureBaseTTSService):
                 yield TTSStoppedFrame()
 
             except Exception as e:
-                logger.error(f"{self} error during synthesis: {e}")
+                logger.error(f"{self} exception: {e}")
+                yield ErrorFrame(error=f"{self} error: {e}")
                 yield TTSStoppedFrame()
                 # Could add reconnection logic here if needed
                 return
 
         except Exception as e:
             logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
 
 class AzureHttpTTSService(AzureBaseTTSService):
@@ -439,3 +441,4 @@ class AzureHttpTTSService(AzureBaseTTSService):
             logger.warning(f"Speech synthesis canceled: {cancellation_details.reason}")
             if cancellation_details.reason == CancellationReason.Error:
                 logger.error(f"{self} error: {cancellation_details.error_details}")
+                yield ErrorFrame(error=f"{self} error: {cancellation_details.error_details}")

src/pipecat/services/cartesia/stt.py

@@ -20,6 +20,7 @@ from loguru import logger
 from pipecat.frames.frames import (
     CancelFrame,
     EndFrame,
+    ErrorFrame,
     Frame,
     InterimTranscriptionFrame,
     StartFrame,
@@ -275,7 +276,8 @@ class CartesiaSTTService(WebsocketSTTService):
             self._websocket = await websocket_connect(ws_url, additional_headers=headers)
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"{self}: unable to connect to Cartesia: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
     async def _disconnect_websocket(self):
         try:
@@ -284,6 +286,7 @@ class CartesiaSTTService(WebsocketSTTService):
                 await self._websocket.close()
         except Exception as e:
             logger.error(f"{self} error closing websocket: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._websocket = None
             await self._call_event_handler("on_disconnected")
@@ -315,7 +318,9 @@ class CartesiaSTTService(WebsocketSTTService):
                 await self._on_transcript(data)
 
             elif data["type"] == "error":
-                logger.error(f"Cartesia error: {data.get('message', 'Unknown error')}")
+                error_msg = data.get("message", "Unknown error")
+                logger.error(f"Cartesia error: {error_msg}")
+                await self.push_error(ErrorFrame(error=error_msg))
 
     @traced_stt
     async def _handle_transcription(

src/pipecat/services/cartesia/tts.py

@@ -10,7 +10,8 @@ import base64
 import json
 import uuid
 import warnings
-from typing import AsyncGenerator, List, Literal, Optional, Union
+from enum import Enum
+from typing import AsyncGenerator, List, Literal, Optional
 
 from loguru import logger
 from pydantic import BaseModel, Field
@@ -125,6 +126,72 @@ def language_to_cartesia_language(language: Language) -> Optional[str]:
     return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
 
 
+class CartesiaEmotion(str, Enum):
+    """Predefined Emotions supported by Cartesia."""
+
+    # Primary emotions supported by Cartesia
+    NEUTRAL = "neutral"
+    ANGRY = "angry"
+    EXCITED = "excited"
+    CONTENT = "content"
+    SAD = "sad"
+    SCARED = "scared"
+    # Additional emotions supported by Cartesia
+    HAPPY = "happy"
+    ENTHUSIASTIC = "enthusiastic"
+    ELATED = "elated"
+    EUPHORIC = "euphoric"
+    TRIUMPHANT = "triumphant"
+    AMAZED = "amazed"
+    SURPRISED = "surprised"
+    FLIRTATIOUS = "flirtatious"
+    JOKING_COMEDIC = "joking/comedic"
+    CURIOUS = "curious"
+    PEACEFUL = "peaceful"
+    SERENE = "serene"
+    CALM = "calm"
+    GRATEFUL = "grateful"
+    AFFECTIONATE = "affectionate"
+    TRUST = "trust"
+    SYMPATHETIC = "sympathetic"
+    ANTICIPATION = "anticipation"
+    MYSTERIOUS = "mysterious"
+    MAD = "mad"
+    OUTRAGED = "outraged"
+    FRUSTRATED = "frustrated"
+    AGITATED = "agitated"
+    THREATENED = "threatened"
+    DISGUSTED = "disgusted"
+    CONTEMPT = "contempt"
+    ENVIOUS = "envious"
+    SARCASTIC = "sarcastic"
+    IRONIC = "ironic"
+    DEJECTED = "dejected"
+    MELANCHOLIC = "melancholic"
+    DISAPPOINTED = "disappointed"
+    HURT = "hurt"
+    GUILTY = "guilty"
+    BORED = "bored"
+    TIRED = "tired"
+    REJECTED = "rejected"
+    NOSTALGIC = "nostalgic"
+    WISTFUL = "wistful"
+    APOLOGETIC = "apologetic"
+    HESITANT = "hesitant"
+    INSECURE = "insecure"
+    CONFUSED = "confused"
+    RESIGNED = "resigned"
+    ANXIOUS = "anxious"
+    PANICKED = "panicked"
+    ALARMED = "alarmed"
+    PROUD = "proud"
+    CONFIDENT = "confident"
+    DISTANT = "distant"
+    SKEPTICAL = "skeptical"
+    CONTEMPLATIVE = "contemplative"
+    DETERMINED = "determined"
+
+
 class CartesiaTTSService(AudioContextWordTTSService):
     """Cartesia TTS service with WebSocket streaming and word timestamps.
 
@@ -182,6 +249,10 @@ class CartesiaTTSService(AudioContextWordTTSService):
             container: Audio container format.
             params: Additional input parameters for voice customization.
             text_aggregator: Custom text aggregator for processing input text.
+
+                .. deprecated:: 0.0.95
+                    Use an LLMTextProcessor before the TTSService for custom text aggregation.
+
             aggregate_sentences: Whether to aggregate sentences within the TTSService.
             **kwargs: Additional arguments passed to the parent service.
         """
@@ -200,10 +271,18 @@ class CartesiaTTSService(AudioContextWordTTSService):
             push_text_frames=False,
             pause_frame_processing=True,
             sample_rate=sample_rate,
-            text_aggregator=text_aggregator or SkipTagsAggregator([("<spell>", "</spell>")]),
+            text_aggregator=text_aggregator,
             **kwargs,
         )
 
+        if not text_aggregator:
+            # Always skip tags added for spelled-out text
+            # Note: This is primarily to support backwards compatibility.
+            #    The preferred way of taking advantage of Cartesia SSML Tags is
+            #    to use an LLMTextProcessor and/or a text_transformer to identify
+            #    and insert these tags for the purpose of the TTS service alone.
+            self._text_aggregator = SkipTagsAggregator([("<spell>", "</spell>")])
+
         params = params or CartesiaTTSService.InputParams()
 
         self._api_key = api_key
@@ -257,6 +336,27 @@ class CartesiaTTSService(AudioContextWordTTSService):
         """
         return language_to_cartesia_language(language)
 
+    # A set of Cartesia-specific helpers for text transformations
+    def SPELL(text: str) -> str:
+        """Wrap text in Cartesia spell tag."""
+        return f"<spell>{text}</spell>"
+
+    def EMOTION_TAG(emotion: CartesiaEmotion) -> str:
+        """Convenience method to create an emotion tag."""
+        return f'<emotion value="{emotion}" />'
+
+    def PAUSE_TAG(seconds: float) -> str:
+        """Convenience method to create a pause tag."""
+        return f'<break time="{seconds}s" />'
+
+    def VOLUME_TAG(volume: float) -> str:
+        """Convenience method to create a volume tag."""
+        return f'<volume ratio="{volume}" />'
+
+    def SPEED_TAG(speed: float) -> str:
+        """Convenience method to create a speed tag."""
+        return f'<speed ratio="{speed}" />'
+
     def _is_cjk_language(self, language: str) -> bool:
         """Check if the given language is CJK (Chinese, Japanese, Korean).
 
@@ -397,7 +497,8 @@ class CartesiaTTSService(AudioContextWordTTSService):
             )
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"{self} initialization error: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
 
@@ -409,7 +510,8 @@ class CartesiaTTSService(AudioContextWordTTSService):
                 logger.debug("Disconnecting from Cartesia")
                 await self._websocket.close()
         except Exception as e:
-            logger.error(f"{self} error closing websocket: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._context_id = None
             self._websocket = None
@@ -465,7 +567,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
                 logger.error(f"{self} error: {msg}")
                 await self.push_frame(TTSStoppedFrame())
                 await self.stop_all_metrics()
-                await self.push_error(ErrorFrame(f"{self} error: {msg['error']}"))
+                await self.push_error(ErrorFrame(error=f"{self} error: {msg['error']}"))
                 self._context_id = None
             else:
                 logger.error(f"{self} error, unknown message type: {msg}")
@@ -506,7 +608,8 @@ class CartesiaTTSService(AudioContextWordTTSService):
                 await self._get_websocket().send(msg)
                 await self.start_tts_usage_metrics(text)
             except Exception as e:
-                logger.error(f"{self} error sending message: {e}")
+                logger.error(f"{self} exception: {e}")
+                yield ErrorFrame(error=f"{self} error: {e}")
                 yield TTSStoppedFrame()
                 await self._disconnect()
                 await self._connect()
@@ -514,6 +617,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
             yield None
         except Exception as e:
             logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
 
 class CartesiaHttpTTSService(TTSService):
@@ -705,7 +809,7 @@ class CartesiaHttpTTSService(TTSService):
                 if response.status != 200:
                     error_text = await response.text()
                     logger.error(f"Cartesia API error: {error_text}")
-                    await self.push_error(ErrorFrame(f"Cartesia API error: {error_text}"))
+                    await self.push_error(ErrorFrame(error=f"Cartesia API error: {error_text}"))
                     raise Exception(f"Cartesia API returned status {response.status}: {error_text}")
 
                 audio_data = await response.read()
@@ -722,7 +826,7 @@ class CartesiaHttpTTSService(TTSService):
 
         except Exception as e:
             logger.error(f"{self} exception: {e}")
-            await self.push_error(ErrorFrame(f"Error generating TTS: {e}"))
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             await self.stop_ttfb_metrics()
             yield TTSStoppedFrame()

src/pipecat/services/deepgram/flux/stt.py

@@ -6,9 +6,12 @@
 
 """Deepgram Flux speech-to-text service implementation."""
 
+import asyncio
 import json
+import time
 from enum import Enum
 from typing import Any, AsyncGenerator, Dict, Optional
+from urllib.parse import urlencode
 
 from loguru import logger
 from pydantic import BaseModel
@@ -93,6 +96,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
             mip_opt_out: Optional. Opts out requests from the Deepgram Model Improvement Program
                 (default False).
             tag: List of tags to label requests for identification during usage reporting.
+            min_confidence: Optional. Minimum confidence required confidence to create a TranscriptionFrame
         """
 
         eager_eot_threshold: Optional[float] = None
@@ -101,6 +105,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         keyterm: list = []
         mip_opt_out: Optional[bool] = None
         tag: list = []
+        min_confidence: Optional[float] = None  # New parameter
 
     def __init__(
         self,
@@ -162,6 +167,13 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         self._register_event_handler("on_end_of_turn")
         self._register_event_handler("on_eager_end_of_turn")
         self._register_event_handler("on_update")
+        self._connection_established_event = asyncio.Event()
+        # Watchdog task to prevent dangling tasks
+        # If we stop sending audio to Flux after we have received that the User has started speaking
+        # we never receive the user stopped speaking event unless we resume sending audio to it.
+        self._last_stt_time = None
+        self._watchdog_task = None
+        self._user_is_speaking = False
 
     async def _connect(self):
         """Connect to WebSocket and start background tasks.
@@ -171,9 +183,6 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         """
         await self._connect_websocket()
 
-        if self._websocket and not self._receive_task:
-            self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
-
     async def _disconnect(self):
         """Disconnect from WebSocket and clean up tasks.
 
@@ -181,20 +190,33 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         and cleans up resources to prevent memory leaks.
         """
         try:
-            # Cancel background tasks BEFORE closing websocket
-            if self._receive_task:
-                await self.cancel_task(self._receive_task, timeout=2.0)
-                self._receive_task = None
-
-            # Now close the websocket
             await self._disconnect_websocket()
-
         except Exception as e:
-            logger.error(f"Error during disconnect: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             # Reset state only after everything is cleaned up
             self._websocket = None
 
+    async def _send_silence(self, duration_secs: float = 0.5):
+        """Send a block of silence of the specified duration (default 500 ms)."""
+        sample_width = 2  # bytes per sample for 16-bit PCM
+        num_channels = 1  # mono
+        num_samples = int(self.sample_rate * duration_secs)
+        silence = b"\x00" * (num_samples * sample_width * num_channels)
+        await self._websocket.send(silence)
+
+    async def _watchdog_task_handler(self):
+        while self._websocket and self._websocket.state is State.OPEN:
+            now = time.monotonic()
+            # More than 500 ms without sending new audio to Flux
+            if self._user_is_speaking and self._last_stt_time and now - self._last_stt_time > 0.5:
+                logger.warning("Sending silence to Flux to prevent dangling task")
+                await self._send_silence()
+                self._last_stt_time = time.monotonic()
+            # check every 100ms
+            await asyncio.sleep(0.1)
+
     async def _connect_websocket(self):
         """Establish WebSocket connection to API.
 
@@ -206,14 +228,31 @@ class DeepgramFluxSTTService(WebsocketSTTService):
             if self._websocket and self._websocket.state is State.OPEN:
                 return
 
+            self._connection_established_event.clear()
+            self._user_is_speaking = False
             self._websocket = await websocket_connect(
                 self._websocket_url,
                 additional_headers={"Authorization": f"Token {self._api_key}"},
             )
+
+            # Creating the receiver task
+            if not self._receive_task:
+                self._receive_task = self.create_task(
+                    self._receive_task_handler(self._report_error)
+                )
+
+            # Creating the watchdog task
+            if not self._watchdog_task:
+                self._watchdog_task = self.create_task(self._watchdog_task_handler())
+
+            # Now wait for the connection established event
+            logger.debug("WebSocket connected, waiting for server confirmation...")
+            await self._connection_established_event.wait()
             logger.debug("Connected to Deepgram Flux Websocket")
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"{self} initialization error: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
 
@@ -224,6 +263,16 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         metrics collection. Handles disconnection errors gracefully.
         """
         try:
+            # Cancel background tasks BEFORE closing websocket
+            if self._receive_task:
+                await self.cancel_task(self._receive_task, timeout=2.0)
+                self._receive_task = None
+            if self._watchdog_task:
+                await self.cancel_task(self._watchdog_task, timeout=2.0)
+                self._watchdog_task = None
+                self._last_stt_time = None
+
+            self._connection_established_event.clear()
             await self.stop_all_metrics()
 
             if self._websocket:
@@ -232,6 +281,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
                 await self._websocket.close()
         except Exception as e:
             logger.error(f"{self} error closing websocket: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._websocket = None
             await self._call_event_handler("on_disconnected")
@@ -285,11 +335,11 @@ class DeepgramFluxSTTService(WebsocketSTTService):
 
         # Add keyterm parameters (can have multiple)
         for keyterm in self._params.keyterm:
-            url_params.append(f"keyterm={keyterm}")
+            url_params.append(urlencode({"keyterm": keyterm}))
 
         # Add tag parameters (can have multiple)
         for tag_value in self._params.tag:
-            url_params.append(f"tag={tag_value}")
+            url_params.append(urlencode({"tag": tag_value}))
 
         self._websocket_url = f"{self._url}?{'&'.join(url_params)}"
         await self._connect()
@@ -332,14 +382,15 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         """
         if not self._websocket:
             logger.error("Not connected to Deepgram Flux.")
-            yield ErrorFrame("Not connected to Deepgram Flux.", fatal=True)
+            yield ErrorFrame("Not connected to Deepgram Flux.")
             return
 
         try:
-            await self._websocket.send(audio)
+            self._last_stt_time = time.monotonic()
+            await self.send_with_retry(audio, self._report_error)
         except Exception as e:
-            logger.error(f"Failed to send audio to Flux: {e}")
-            yield ErrorFrame(f"Failed to send audio to Flux:  {e}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
             return
 
         yield None
@@ -416,7 +467,8 @@ class DeepgramFluxSTTService(WebsocketSTTService):
                     # Skip malformed messages
                     continue
                 except Exception as e:
-                    logger.error(f"Error processing message: {e}")
+                    logger.error(f"{self} exception: {e}")
+                    await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
                     # Error will be handled inside WebsocketService->_receive_task_handler
                     raise
             else:
@@ -458,6 +510,8 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         transcription processing.
         """
         logger.info("Connected to Flux - ready to stream audio")
+        # Notify connection is established
+        self._connection_established_event.set()
 
     async def _handle_fatal_error(self, data: Dict[str, Any]):
         """Handle fatal error messages from Deepgram Flux.
@@ -525,6 +579,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
             transcript: maybe the first few words of the turn.
         """
         logger.debug("User started speaking")
+        self._user_is_speaking = True
         await self.push_interruption_task_frame_and_wait()
         await self.broadcast_frame(UserStartedSpeakingFrame)
         await self.start_metrics()
@@ -545,6 +600,22 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         logger.trace(f"Received event TurnResumed: {event}")
         await self._call_event_handler("on_turn_resumed")
 
+    def _calculate_average_confidence(self, transcript_data) -> Optional[float]:
+        """Calculate the average confidence from transcript data.
+
+        Return None if the data is missing or invalid.
+        """
+        # Example: Assume transcript_data has a list of words with confidence
+        words = transcript_data.get("words")
+        if not words or not isinstance(words, list):
+            return None
+        confidences = [
+            w.get("confidence") for w in words if isinstance(w.get("confidence"), (float, int))
+        ]
+        if not confidences:
+            return None
+        return sum(confidences) / len(confidences)
+
     async def _handle_end_of_turn(self, transcript: str, data: Dict[str, Any]):
         """Handle EndOfTurn events from Deepgram Flux.
 
@@ -564,16 +635,26 @@ class DeepgramFluxSTTService(WebsocketSTTService):
             data: The TurnInfo message data containing event type, transcript and some extra metadata.
         """
         logger.debug("User stopped speaking")
+        self._user_is_speaking = False
 
-        await self.push_frame(
-            TranscriptionFrame(
-                transcript,
-                self._user_id,
-                time_now_iso8601(),
-                self._language,
-                result=data,
+        # Compute the average confidence
+        average_confidence = self._calculate_average_confidence(data)
+
+        if not self._params.min_confidence or average_confidence > self._params.min_confidence:
+            await self.push_frame(
+                TranscriptionFrame(
+                    transcript,
+                    self._user_id,
+                    time_now_iso8601(),
+                    self._language,
+                    result=data,
+                )
             )
-        )
+        else:
+            logger.warning(
+                f"Transcription confidence below min_confidence threshold: {average_confidence}"
+            )
+
         await self._handle_transcription(transcript, True, self._language)
         await self.stop_processing_metrics()
         await self.push_frame(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM)

src/pipecat/services/deepgram/stt.py

@@ -256,7 +256,7 @@ class DeepgramSTTService(STTService):
     async def _on_error(self, *args, **kwargs):
         error: ErrorResponse = kwargs["error"]
         logger.warning(f"{self} connection error, will retry: {error}")
-        await self.push_error(ErrorFrame(f"{error}"))
+        await self.push_error(ErrorFrame(error=f"{error}"))
         await self.stop_all_metrics()
         # NOTE(aleix): we don't disconnect (i.e. call finish on the connection)
         # because this triggers more errors internally in the Deepgram SDK. So,

src/pipecat/services/deepgram/stt_sagemaker.py

@@ -0,0 +1,447 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Deepgram speech-to-text service for AWS SageMaker.
+
+This module provides a Pipecat STT service that connects to Deepgram models
+deployed on AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for
+low-latency real-time transcription with support for interim results, multiple
+languages, and various Deepgram features.
+"""
+
+import asyncio
+import json
+from typing import AsyncGenerator, Optional
+
+from loguru import logger
+
+from pipecat.frames.frames import (
+    CancelFrame,
+    EndFrame,
+    ErrorFrame,
+    Frame,
+    InterimTranscriptionFrame,
+    StartFrame,
+    TranscriptionFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient
+from pipecat.services.stt_service import STTService
+from pipecat.transcriptions.language import Language
+from pipecat.utils.time import time_now_iso8601
+from pipecat.utils.tracing.service_decorators import traced_stt
+
+try:
+    from deepgram import LiveOptions
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use DeepgramSageMakerSTTService, you need to `pip install pipecat-ai[deepgram,sagemaker]`."
+    )
+    raise Exception(f"Missing module: {e}")
+
+
+class DeepgramSageMakerSTTService(STTService):
+    """Deepgram speech-to-text service for AWS SageMaker.
+
+    Provides real-time speech recognition using Deepgram models deployed on
+    AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for low-latency
+    transcription with support for interim results, speaker diarization, and
+    multiple languages.
+
+    Requirements:
+
+    - AWS credentials configured (via environment variables, AWS CLI, or instance metadata)
+    - A deployed SageMaker endpoint with Deepgram model: https://developers.deepgram.com/docs/deploy-amazon-sagemaker
+    - Deepgram SDK for LiveOptions configuration
+
+    Example::
+
+        stt = DeepgramSageMakerSTTService(
+            endpoint_name="my-deepgram-endpoint",
+            region="us-east-2",
+            live_options=LiveOptions(
+                model="nova-3",
+                language="en",
+                interim_results=True,
+                punctuate=True,
+            ),
+        )
+    """
+
+    def __init__(
+        self,
+        *,
+        endpoint_name: str,
+        region: str,
+        sample_rate: Optional[int] = None,
+        live_options: Optional[LiveOptions] = None,
+        **kwargs,
+    ):
+        """Initialize the Deepgram SageMaker STT service.
+
+        Args:
+            endpoint_name: Name of the SageMaker endpoint with Deepgram model
+                deployed (e.g., "my-deepgram-nova-3-endpoint").
+            region: AWS region where the endpoint is deployed (e.g., "us-east-2").
+            sample_rate: Audio sample rate in Hz. If None, uses value from
+                live_options or defaults to the value from StartFrame.
+            live_options: Deepgram LiveOptions for detailed configuration. If None,
+                uses sensible defaults (nova-3 model, English, interim results enabled).
+            **kwargs: Additional arguments passed to the parent STTService.
+        """
+        sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
+        super().__init__(sample_rate=sample_rate, **kwargs)
+
+        self._endpoint_name = endpoint_name
+        self._region = region
+
+        # Create default options similar to DeepgramSTTService
+        default_options = LiveOptions(
+            encoding="linear16",
+            language=Language.EN,
+            model="nova-3",
+            channels=1,
+            interim_results=True,
+            punctuate=True,
+        )
+
+        # Merge with provided options
+        merged_options = default_options.to_dict()
+        if live_options:
+            default_model = default_options.model
+            merged_options.update(live_options.to_dict())
+            # Handle the "None" string bug from deepgram-sdk
+            if "model" in merged_options and merged_options["model"] == "None":
+                merged_options["model"] = default_model
+
+        # Convert Language enum to string if needed
+        if "language" in merged_options and isinstance(merged_options["language"], Language):
+            merged_options["language"] = merged_options["language"].value
+
+        self.set_model_name(merged_options["model"])
+        self._settings = merged_options
+
+        self._client: Optional[SageMakerBidiClient] = None
+        self._response_task: Optional[asyncio.Task] = None
+        self._keepalive_task: Optional[asyncio.Task] = None
+
+    def can_generate_metrics(self) -> bool:
+        """Check if this service can generate processing metrics.
+
+        Returns:
+            True, as Deepgram SageMaker service supports metrics generation.
+        """
+        return True
+
+    async def set_model(self, model: str):
+        """Set the Deepgram model and reconnect.
+
+        Disconnects from the current session, updates the model setting, and
+        establishes a new connection with the updated model.
+
+        Args:
+            model: The Deepgram model name to use (e.g., "nova-3").
+        """
+        await super().set_model(model)
+        logger.info(f"Switching STT model to: [{model}]")
+        self._settings["model"] = model
+        await self._disconnect()
+        await self._connect()
+
+    async def set_language(self, language: Language):
+        """Set the recognition language and reconnect.
+
+        Disconnects from the current session, updates the language setting, and
+        establishes a new connection with the updated language.
+
+        Args:
+            language: The language to use for speech recognition (e.g., Language.EN,
+                Language.ES).
+        """
+        logger.info(f"Switching STT language to: [{language}]")
+        self._settings["language"] = language
+        await self._disconnect()
+        await self._connect()
+
+    async def start(self, frame: StartFrame):
+        """Start the Deepgram SageMaker STT service.
+
+        Args:
+            frame: The start frame containing initialization parameters.
+        """
+        await super().start(frame)
+        self._settings["sample_rate"] = self.sample_rate
+        await self._connect()
+
+    async def stop(self, frame: EndFrame):
+        """Stop the Deepgram SageMaker STT service.
+
+        Args:
+            frame: The end frame.
+        """
+        await super().stop(frame)
+        await self._disconnect()
+
+    async def cancel(self, frame: CancelFrame):
+        """Cancel the Deepgram SageMaker STT service.
+
+        Args:
+            frame: The cancel frame.
+        """
+        await super().cancel(frame)
+        await self._disconnect()
+
+    async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
+        """Send audio data to Deepgram for transcription.
+
+        Args:
+            audio: Raw audio bytes to transcribe.
+
+        Yields:
+            Frame: None (transcription results come via BiDi stream callbacks).
+        """
+        if self._client and self._client.is_active:
+            try:
+                await self._client.send_audio_chunk(audio)
+            except Exception as e:
+                logger.error(f"Error sending audio to SageMaker: {e}")
+                await self.push_error(ErrorFrame(error=f"SageMaker STT error: {e}"))
+        yield None
+
+    async def _connect(self):
+        """Connect to the SageMaker endpoint and start the BiDi session.
+
+        Builds the Deepgram query string from settings, creates the BiDi client,
+        starts the streaming session, and launches background tasks for processing
+        responses and sending KeepAlive messages.
+        """
+        logger.debug("Connecting to Deepgram on SageMaker...")
+
+        # Update sample rate in settings
+        self._settings["sample_rate"] = self.sample_rate
+
+        # Build query string from settings, converting booleans to strings
+        query_params = {}
+        for key, value in self._settings.items():
+            if value is not None:
+                # Convert boolean values to lowercase strings for Deepgram API
+                if isinstance(value, bool):
+                    query_params[key] = str(value).lower()
+                else:
+                    query_params[key] = str(value)
+
+        query_string = "&".join(f"{k}={v}" for k, v in query_params.items())
+
+        # Create BiDi client
+        self._client = SageMakerBidiClient(
+            endpoint_name=self._endpoint_name,
+            region=self._region,
+            model_invocation_path="v1/listen",
+            model_query_string=query_string,
+        )
+
+        try:
+            # Start the session
+            await self._client.start_session()
+
+            # Start processing responses in the background
+            self._response_task = self.create_task(self._process_responses())
+
+            # Start keepalive task to maintain connection
+            self._keepalive_task = self.create_task(self._send_keepalive())
+
+            logger.debug("Connected to Deepgram on SageMaker")
+            await self._call_event_handler("on_connected")
+
+        except Exception as e:
+            logger.error(f"Failed to connect to SageMaker: {e}")
+            await self.push_error(ErrorFrame(error=f"SageMaker connection error: {e}"))
+            await self._call_event_handler("on_connection_error", str(e))
+
+    async def _disconnect(self):
+        """Disconnect from the SageMaker endpoint.
+
+        Sends a CloseStream message to Deepgram, cancels background tasks
+        (KeepAlive and response processing), and closes the BiDi session.
+        Safe to call multiple times.
+        """
+        if self._client and self._client.is_active:
+            logger.debug("Disconnecting from Deepgram on SageMaker...")
+
+            # Send CloseStream message to Deepgram
+            try:
+                await self._client.send_json({"type": "CloseStream"})
+            except Exception as e:
+                logger.warning(f"Failed to send CloseStream message: {e}")
+
+            # Cancel keepalive task
+            if self._keepalive_task and not self._keepalive_task.done():
+                await self.cancel_task(self._keepalive_task)
+
+            # Cancel response processing task
+            if self._response_task and not self._response_task.done():
+                await self.cancel_task(self._response_task)
+
+            # Close the BiDi session
+            await self._client.close_session()
+
+            logger.debug("Disconnected from Deepgram on SageMaker")
+            await self._call_event_handler("on_disconnected")
+
+    async def _send_keepalive(self):
+        """Send periodic KeepAlive messages to maintain the connection.
+
+        Sends a KeepAlive JSON message to Deepgram every 5 seconds while the
+        connection is active. This prevents the connection from timing out during
+        periods of silence.
+        """
+        while self._client and self._client.is_active:
+            await asyncio.sleep(5)
+            if self._client and self._client.is_active:
+                try:
+                    await self._client.send_json({"type": "KeepAlive"})
+                except Exception as e:
+                    logger.warning(f"Failed to send KeepAlive: {e}")
+
+    async def _process_responses(self):
+        """Process streaming responses from Deepgram on SageMaker.
+
+        Continuously receives responses from the BiDi stream, decodes the payload,
+        parses JSON responses from Deepgram, and processes transcription results.
+        Runs as a background task until the connection is closed or cancelled.
+        """
+        try:
+            while self._client and self._client.is_active:
+                result = await self._client.receive_response()
+
+                if result is None:
+                    break
+
+                # Check if this is a PayloadPart with bytes
+                if hasattr(result, "value") and hasattr(result.value, "bytes_"):
+                    if result.value.bytes_:
+                        response_data = result.value.bytes_.decode("utf-8")
+
+                        try:
+                            # Parse JSON response from Deepgram
+                            parsed = json.loads(response_data)
+
+                            # Extract and process transcript if available
+                            if "channel" in parsed:
+                                await self._handle_transcript_response(parsed)
+
+                        except json.JSONDecodeError:
+                            logger.warning(f"Non-JSON response: {response_data}")
+
+        except asyncio.CancelledError:
+            logger.debug("Response processor cancelled")
+        except Exception as e:
+            logger.error(f"Error processing responses: {e}", exc_info=True)
+            await self.push_error(ErrorFrame(error=f"SageMaker response error: {e}"))
+        finally:
+            logger.debug("Response processor stopped")
+
+    async def _handle_transcript_response(self, parsed: dict):
+        """Handle a transcript response from Deepgram.
+
+        Extracts the transcript text, determines if it's final or interim, extracts
+        language information, and pushes the appropriate frame (TranscriptionFrame
+        or InterimTranscriptionFrame) downstream.
+
+        Args:
+            parsed: The parsed JSON response from Deepgram containing channel,
+                alternatives, transcript, and metadata.
+        """
+        alternatives = parsed.get("channel", {}).get("alternatives", [])
+        if not alternatives or not alternatives[0].get("transcript"):
+            return
+
+        transcript = alternatives[0]["transcript"]
+        if not transcript.strip():
+            return
+
+        # Stop TTFB metrics on first transcript
+        await self.stop_ttfb_metrics()
+
+        is_final = parsed.get("is_final", False)
+        speech_final = parsed.get("speech_final", False)
+
+        # Extract language if available
+        language = None
+        if alternatives[0].get("languages"):
+            language = alternatives[0]["languages"][0]
+            language = Language(language)
+
+        if is_final and speech_final:
+            # Final transcription
+            await self.push_frame(
+                TranscriptionFrame(
+                    transcript,
+                    self._user_id,
+                    time_now_iso8601(),
+                    language,
+                    result=parsed,
+                )
+            )
+            await self._handle_transcription(transcript, is_final, language)
+            await self.stop_processing_metrics()
+        else:
+            # Interim transcription
+            await self.push_frame(
+                InterimTranscriptionFrame(
+                    transcript,
+                    self._user_id,
+                    time_now_iso8601(),
+                    language,
+                    result=parsed,
+                )
+            )
+
+    @traced_stt
+    async def _handle_transcription(
+        self, transcript: str, is_final: bool, language: Optional[Language] = None
+    ):
+        """Handle a transcription result with tracing.
+
+        This method is decorated with @traced_stt for observability and tracing
+        integration. The actual transcription processing is handled by the parent
+        class and observers.
+
+        Args:
+            transcript: The transcribed text.
+            is_final: Whether this is a final transcription result.
+            language: The detected language of the transcription, if available.
+        """
+        pass
+
+    async def start_metrics(self):
+        """Start TTFB and processing metrics collection."""
+        await self.start_ttfb_metrics()
+        await self.start_processing_metrics()
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process frames with Deepgram SageMaker-specific handling.
+
+        Args:
+            frame: The frame to process.
+            direction: The direction of frame processing.
+        """
+        await super().process_frame(frame, direction)
+
+        # Start metrics when user starts speaking (if VAD is not provided by Deepgram)
+        if isinstance(frame, UserStartedSpeakingFrame):
+            await self.start_metrics()
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            # Send finalize message to Deepgram when user stops speaking
+            # This tells Deepgram to flush any remaining audio and return final results
+            if self._client and self._client.is_active:
+                try:
+                    await self._client.send_json({"type": "Finalize"})
+                except Exception as e:
+                    logger.warning(f"Error sending Finalize message: {e}")

src/pipecat/services/deepgram/tts.py

@@ -116,8 +116,8 @@ class DeepgramTTSService(TTSService):
             yield TTSStoppedFrame()
 
         except Exception as e:
-            logger.exception(f"{self} exception: {e}")
-            yield ErrorFrame(f"Error getting audio: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
 
 class DeepgramHttpTTSService(TTSService):

src/pipecat/services/elevenlabs/stt.py

@@ -11,19 +11,43 @@ using segmented audio processing. The service uploads audio files and receives
 transcription results directly.
 """
 
+import base64
 import io
+import json
+from enum import Enum
 from typing import AsyncGenerator, Optional
 
 import aiohttp
 from loguru import logger
 from pydantic import BaseModel
 
-from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
-from pipecat.services.stt_service import SegmentedSTTService
+from pipecat.frames.frames import (
+    CancelFrame,
+    EndFrame,
+    ErrorFrame,
+    Frame,
+    InterimTranscriptionFrame,
+    StartFrame,
+    TranscriptionFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.stt_service import SegmentedSTTService, WebsocketSTTService
 from pipecat.transcriptions.language import Language, resolve_language
 from pipecat.utils.time import time_now_iso8601
 from pipecat.utils.tracing.service_decorators import traced_stt
 
+try:
+    from websockets.asyncio.client import connect as websocket_connect
+    from websockets.protocol import State
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use ElevenLabs Realtime STT, you need to `pip install pipecat-ai[elevenlabs]`."
+    )
+    raise Exception(f"Missing module: {e}")
+
 
 def language_to_elevenlabs_language(language: Language) -> Optional[str]:
     """Convert a Language enum to ElevenLabs language code.
@@ -327,5 +351,512 @@ class ElevenLabsSTTService(SegmentedSTTService):
                 )
 
         except Exception as e:
-            logger.error(f"ElevenLabs STT error: {e}")
-            yield ErrorFrame(f"ElevenLabs STT error: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
+
+
+def audio_format_from_sample_rate(sample_rate: int) -> str:
+    """Get the appropriate audio format string for a given sample rate.
+
+    Args:
+        sample_rate: The audio sample rate in Hz.
+
+    Returns:
+        The ElevenLabs audio format string.
+    """
+    match sample_rate:
+        case 8000:
+            return "pcm_8000"
+        case 16000:
+            return "pcm_16000"
+        case 22050:
+            return "pcm_22050"
+        case 24000:
+            return "pcm_24000"
+        case 44100:
+            return "pcm_44100"
+        case 48000:
+            return "pcm_48000"
+    logger.warning(
+        f"ElevenLabsRealtimeSTTService: No audio format available for {sample_rate} sample rate, using pcm_16000"
+    )
+    return "pcm_16000"
+
+
+class CommitStrategy(str, Enum):
+    """Commit strategies for transcript segmentation."""
+
+    MANUAL = "manual"
+    VAD = "vad"
+
+
+class ElevenLabsRealtimeSTTService(WebsocketSTTService):
+    """Speech-to-text service using ElevenLabs' Realtime WebSocket API.
+
+    This service uses ElevenLabs' Realtime Speech-to-Text API to perform transcription
+    with ultra-low latency. It supports both partial (interim) and committed (final)
+    transcripts, and can use either manual commit control or automatic Voice Activity
+    Detection (VAD) for segment boundaries.
+
+    By default, uses manual commit strategy where Pipecat's VAD controls when to
+    commit transcript segments, providing consistency with other STT services.
+    """
+
+    class InputParams(BaseModel):
+        """Configuration parameters for ElevenLabs Realtime STT API.
+
+        Parameters:
+            language_code: ISO-639-1 or ISO-639-3 language code. Leave None for auto-detection.
+            commit_strategy: How to segment speech - manual (Pipecat VAD) or vad (ElevenLabs VAD).
+            vad_silence_threshold_secs: Seconds of silence before VAD commits (0.3-3.0).
+                Only used when commit_strategy is VAD. None uses ElevenLabs default.
+            vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive).
+                Only used when commit_strategy is VAD. None uses ElevenLabs default.
+            min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms).
+                Only used when commit_strategy is VAD. None uses ElevenLabs default.
+            min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms).
+                Only used when commit_strategy is VAD. None uses ElevenLabs default.
+            include_timestamps: Whether to include word-level timestamps in transcripts.
+            enable_logging: Whether to enable logging on ElevenLabs' side.
+        """
+
+        language_code: Optional[str] = None
+        commit_strategy: CommitStrategy = CommitStrategy.MANUAL
+        vad_silence_threshold_secs: Optional[float] = None
+        vad_threshold: Optional[float] = None
+        min_speech_duration_ms: Optional[int] = None
+        min_silence_duration_ms: Optional[int] = None
+        include_timestamps: bool = False
+        enable_logging: bool = False
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        base_url: str = "api.elevenlabs.io",
+        model: str = "scribe_v2_realtime",
+        sample_rate: Optional[int] = None,
+        params: Optional[InputParams] = None,
+        **kwargs,
+    ):
+        """Initialize the ElevenLabs Realtime STT service.
+
+        Args:
+            api_key: ElevenLabs API key for authentication.
+            base_url: Base URL for ElevenLabs WebSocket API.
+            model: Model ID for transcription. Defaults to "scribe_v2_realtime".
+            sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate.
+            params: Configuration parameters for the STT service.
+            **kwargs: Additional arguments passed to WebsocketSTTService.
+        """
+        super().__init__(
+            sample_rate=sample_rate,
+            **kwargs,
+        )
+
+        params = params or ElevenLabsRealtimeSTTService.InputParams()
+
+        self._api_key = api_key
+        self._base_url = base_url
+        self._model_id = model
+        self._params = params
+        self._audio_format = ""  # initialized in start()
+        self._receive_task = None
+
+        self._settings = {"language": params.language_code}
+
+    def can_generate_metrics(self) -> bool:
+        """Check if the service can generate processing metrics.
+
+        Returns:
+            True, as ElevenLabs Realtime STT service supports metrics generation.
+        """
+        return True
+
+    async def set_language(self, language: Language):
+        """Set the transcription language.
+
+        Args:
+            language: The language to use for speech-to-text transcription.
+
+        Note:
+            Changing language requires reconnecting to the WebSocket.
+        """
+        logger.info(f"Switching STT language to: [{language}]")
+        new_language = (
+            language_to_elevenlabs_language(language)
+            if isinstance(language, Language)
+            else language
+        )
+        self._params.language_code = new_language
+        self._settings["language"] = new_language
+        # Reconnect with new settings
+        await self._disconnect()
+        await self._connect()
+
+    async def set_model(self, model: str):
+        """Set the STT model.
+
+        Args:
+            model: The model name to use for transcription.
+
+        Note:
+            Changing model requires reconnecting to the WebSocket.
+        """
+        await super().set_model(model)
+        logger.info(f"Switching STT model to: [{model}]")
+        self._model_id = model
+        # Reconnect with new settings
+        await self._disconnect()
+        await self._connect()
+
+    async def start(self, frame: StartFrame):
+        """Start the STT service and establish WebSocket connection.
+
+        Args:
+            frame: Frame indicating service should start.
+        """
+        await super().start(frame)
+        self._audio_format = audio_format_from_sample_rate(self.sample_rate)
+        await self._connect()
+
+    async def stop(self, frame: EndFrame):
+        """Stop the STT service and close WebSocket connection.
+
+        Args:
+            frame: Frame indicating service should stop.
+        """
+        await super().stop(frame)
+        await self._disconnect()
+
+    async def cancel(self, frame: CancelFrame):
+        """Cancel the STT service and close WebSocket connection.
+
+        Args:
+            frame: Frame indicating service should be cancelled.
+        """
+        await super().cancel(frame)
+        await self._disconnect()
+
+    async def start_metrics(self):
+        """Start performance metrics collection for transcription processing."""
+        await self.start_ttfb_metrics()
+        await self.start_processing_metrics()
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames and handle speech events.
+
+        Args:
+            frame: The frame to process.
+            direction: Direction of frame flow in the pipeline.
+        """
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, UserStartedSpeakingFrame):
+            # Start metrics when user starts speaking
+            await self.start_metrics()
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            # Send commit when user stops speaking (manual commit mode)
+            if self._params.commit_strategy == CommitStrategy.MANUAL:
+                if self._websocket and self._websocket.state is State.OPEN:
+                    try:
+                        commit_message = {
+                            "message_type": "input_audio_chunk",
+                            "audio_base_64": "",
+                            "commit": True,
+                            "sample_rate": self.sample_rate,
+                        }
+                        await self._websocket.send(json.dumps(commit_message))
+                        logger.trace("Sent manual commit to ElevenLabs")
+                    except Exception as e:
+                        logger.warning(f"Failed to send commit: {e}")
+
+    async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
+        """Process audio data for speech-to-text transcription.
+
+        Args:
+            audio: Raw audio bytes to transcribe.
+
+        Yields:
+            None - transcription results are handled via WebSocket responses.
+        """
+        # Reconnect if connection is closed
+        if not self._websocket or self._websocket.state is State.CLOSED:
+            await self._connect()
+
+        if self._websocket and self._websocket.state is State.OPEN:
+            try:
+                # Encode audio as base64
+                audio_base64 = base64.b64encode(audio).decode("utf-8")
+
+                # Send audio chunk
+                message = {
+                    "message_type": "input_audio_chunk",
+                    "audio_base_64": audio_base64,
+                    "commit": False,
+                    "sample_rate": self.sample_rate,
+                }
+                await self._websocket.send(json.dumps(message))
+            except Exception as e:
+                logger.error(f"Error sending audio: {e}")
+                yield ErrorFrame(f"ElevenLabs Realtime STT error: {str(e)}")
+
+        yield None
+
+    async def _connect(self):
+        """Establish WebSocket connection to ElevenLabs Realtime STT."""
+        await self._connect_websocket()
+
+        if self._websocket and not self._receive_task:
+            self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
+
+    async def _disconnect(self):
+        """Close WebSocket connection and cleanup tasks."""
+        if self._receive_task:
+            await self.cancel_task(self._receive_task)
+            self._receive_task = None
+
+        await self._disconnect_websocket()
+
+    async def _connect_websocket(self):
+        """Connect to ElevenLabs Realtime STT WebSocket endpoint."""
+        try:
+            if self._websocket and self._websocket.state is State.OPEN:
+                return
+
+            logger.debug("Connecting to ElevenLabs Realtime STT")
+
+            # Build query parameters
+            params = [f"model_id={self._model_id}"]
+
+            if self._params.language_code:
+                params.append(f"language_code={self._params.language_code}")
+
+            params.append(f"audio_format={self._audio_format}")
+            params.append(f"commit_strategy={self._params.commit_strategy.value}")
+
+            # Add optional parameters
+            if self._params.include_timestamps:
+                params.append(f"include_timestamps={str(self._params.include_timestamps).lower()}")
+
+            if self._params.enable_logging:
+                params.append(f"enable_logging={str(self._params.enable_logging).lower()}")
+
+            # Add VAD parameters if using VAD commit strategy and values are specified
+            if self._params.commit_strategy == CommitStrategy.VAD:
+                if self._params.vad_silence_threshold_secs is not None:
+                    params.append(
+                        f"vad_silence_threshold_secs={self._params.vad_silence_threshold_secs}"
+                    )
+                if self._params.vad_threshold is not None:
+                    params.append(f"vad_threshold={self._params.vad_threshold}")
+                if self._params.min_speech_duration_ms is not None:
+                    params.append(f"min_speech_duration_ms={self._params.min_speech_duration_ms}")
+                if self._params.min_silence_duration_ms is not None:
+                    params.append(f"min_silence_duration_ms={self._params.min_silence_duration_ms}")
+
+            ws_url = f"wss://{self._base_url}/v1/speech-to-text/realtime?{'&'.join(params)}"
+
+            headers = {"xi-api-key": self._api_key}
+
+            self._websocket = await websocket_connect(ws_url, additional_headers=headers)
+            await self._call_event_handler("on_connected")
+            logger.debug("Connected to ElevenLabs Realtime STT")
+        except Exception as e:
+            logger.error(f"{self}: unable to connect to ElevenLabs Realtime STT: {e}")
+            await self.push_error(ErrorFrame(f"Connection error: {str(e)}"))
+
+    async def _disconnect_websocket(self):
+        """Disconnect from ElevenLabs Realtime STT WebSocket."""
+        try:
+            if self._websocket and self._websocket.state is State.OPEN:
+                logger.debug("Disconnecting from ElevenLabs Realtime STT")
+                await self._websocket.close()
+        except Exception as e:
+            logger.error(f"{self} error closing websocket: {e}")
+        finally:
+            self._websocket = None
+            await self._call_event_handler("on_disconnected")
+
+    def _get_websocket(self):
+        """Get the current WebSocket connection.
+
+        Returns:
+            The WebSocket connection.
+
+        Raises:
+            Exception: If WebSocket is not connected.
+        """
+        if self._websocket:
+            return self._websocket
+        raise Exception("Websocket not connected")
+
+    async def _process_messages(self):
+        """Process incoming WebSocket messages."""
+        async for message in self._get_websocket():
+            try:
+                data = json.loads(message)
+                await self._process_response(data)
+            except json.JSONDecodeError:
+                logger.warning(f"Received non-JSON message: {message}")
+            except Exception as e:
+                logger.error(f"Error processing message: {e}")
+
+    async def _receive_messages(self):
+        """Continuously receive and process WebSocket messages."""
+        try:
+            await self._process_messages()
+        except Exception as e:
+            logger.warning(f"{self} WebSocket connection closed: {e}")
+            # Connection closed, will reconnect on next audio chunk
+
+    async def _process_response(self, data: dict):
+        """Process a response message from ElevenLabs.
+
+        Args:
+            data: Parsed JSON response data.
+        """
+        message_type = data.get("message_type")
+
+        if message_type == "session_started":
+            logger.debug(f"ElevenLabs session started: {data}")
+
+        elif message_type == "partial_transcript":
+            await self._on_partial_transcript(data)
+
+        elif message_type == "committed_transcript":
+            await self._on_committed_transcript(data)
+
+        elif message_type == "committed_transcript_with_timestamps":
+            await self._on_committed_transcript_with_timestamps(data)
+
+        elif message_type == "error":
+            error_msg = data.get("error", "Unknown error")
+            logger.error(f"ElevenLabs error: {error_msg}")
+            await self.push_error(ErrorFrame(f"Error: {error_msg}"))
+
+        elif message_type == "auth_error":
+            error_msg = data.get("error", "Authentication error")
+            logger.error(f"ElevenLabs auth error: {error_msg}")
+            await self.push_error(ErrorFrame(f"Auth error: {error_msg}"))
+
+        elif message_type == "quota_exceeded_error":
+            error_msg = data.get("error", "Quota exceeded")
+            logger.error(f"ElevenLabs quota exceeded: {error_msg}")
+            await self.push_error(ErrorFrame(f"Quota exceeded: {error_msg}"))
+
+        else:
+            logger.debug(f"Unknown message type: {message_type}")
+
+    async def _on_partial_transcript(self, data: dict):
+        """Handle partial transcript (interim results).
+
+        Args:
+            data: Partial transcript data.
+        """
+        text = data.get("text", "").strip()
+        if not text:
+            return
+
+        await self.stop_ttfb_metrics()
+
+        # Get language if provided
+        language = data.get("language_code")
+
+        logger.trace(f"Partial transcript: [{text}]")
+
+        await self.push_frame(
+            InterimTranscriptionFrame(
+                text,
+                self._user_id,
+                time_now_iso8601(),
+                language,
+                result=data,
+            )
+        )
+
+    @traced_stt
+    async def _handle_transcription(
+        self, transcript: str, is_final: bool, language: Optional[str] = None
+    ):
+        """Handle a transcription result with tracing."""
+        pass
+
+    async def _on_committed_transcript(self, data: dict):
+        """Handle committed transcript (final results).
+
+        Args:
+            data: Committed transcript data.
+        """
+        # If timestamps are enabled, skip this message and wait for the
+        # committed_transcript_with_timestamps message which contains all the data
+        if self._params.include_timestamps:
+            return
+
+        text = data.get("text", "").strip()
+        if not text:
+            return
+
+        await self.stop_ttfb_metrics()
+        await self.stop_processing_metrics()
+
+        # Get language if provided
+        language = data.get("language_code")
+
+        logger.debug(f"Committed transcript: [{text}]")
+
+        await self._handle_transcription(text, True, language)
+
+        await self.push_frame(
+            TranscriptionFrame(
+                text,
+                self._user_id,
+                time_now_iso8601(),
+                language,
+                result=data,
+            )
+        )
+
+    async def _on_committed_transcript_with_timestamps(self, data: dict):
+        """Handle committed transcript with word-level timestamps.
+
+        This message is sent when include_timestamps=true. The result data includes:
+        - text: The transcribed text
+        - language_code: Detected language (if available)
+        - words: Array of word objects with timing information:
+            - text: The word text
+            - start: Start time in seconds
+            - end: End time in seconds
+            - type: "word" or "spacing"
+            - speaker_id: Speaker identifier (if available)
+            - logprob: Log probability score (if available)
+            - characters: Array of character strings (if available)
+
+        Args:
+            data: Committed transcript data with timestamps.
+        """
+        text = data.get("text", "").strip()
+        if not text:
+            return
+
+        await self.stop_ttfb_metrics()
+        await self.stop_processing_metrics()
+
+        # Get language if provided
+        language = data.get("language_code")
+
+        logger.debug(f"Committed transcript with timestamps: [{text}]")
+
+        await self._handle_transcription(text, True, language)
+
+        # This message is sent after committed_transcript when include_timestamps=true.
+        # It contains the full transcript data including text and word-level timestamps.
+        await self.push_frame(
+            TranscriptionFrame(
+                text,
+                self._user_id,
+                time_now_iso8601(),
+                language,
+                result=data,
+            )
+        )

src/pipecat/services/elevenlabs/tts.py

@@ -424,7 +424,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
                         json.dumps({"context_id": self._context_id, "close_context": True})
                     )
             except Exception as e:
-                logger.warning(f"Error closing context for voice settings update: {e}")
+                logger.error(f"{self} exception: {e}")
+                await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._context_id = None
             self._started = False
 
@@ -535,8 +536,9 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
 
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"{self} initialization error: {e}")
+            logger.error(f"{self} exception: {e}")
             self._websocket = None
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             await self._call_event_handler("on_connection_error", f"{e}")
 
     async def _disconnect_websocket(self):
@@ -551,7 +553,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
                 await self._websocket.close()
                 logger.debug("Disconnected from ElevenLabs")
         except Exception as e:
-            logger.error(f"{self} error closing websocket: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._started = False
             self._context_id = None
@@ -581,7 +584,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
                     json.dumps({"context_id": self._context_id, "close_context": True})
                 )
             except Exception as e:
-                logger.error(f"Error closing context on interruption: {e}")
+                logger.error(f"{self} exception: {e}")
+                await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._context_id = None
             self._started = False
             self._partial_word = ""
@@ -736,13 +740,15 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
                 else:
                     await self._send_text(text)
             except Exception as e:
-                logger.error(f"{self} error sending message: {e}")
+                logger.error(f"{self} exception: {e}")
                 yield TTSStoppedFrame()
+                yield ErrorFrame(error=f"{self} error: {e}")
                 self._started = False
                 return
             yield None
         except Exception as e:
             logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
 
 class ElevenLabsHttpTTSService(WordTTSService):
@@ -1085,7 +1091,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
                         logger.warning(f"Failed to parse JSON from stream: {e}")
                         continue
                     except Exception as e:
-                        logger.error(f"Error processing response: {e}", exc_info=True)
+                        logger.error(f"{self} exception: {e}")
+                        yield ErrorFrame(error=f"{self} error: {e}")
                         continue
 
                 # After processing all chunks, emit any remaining partial word
@@ -1109,8 +1116,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
                     self._previous_text = text
 
         except Exception as e:
-            logger.error(f"Error in run_tts: {e}")
-            yield ErrorFrame(error=str(e))
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
         finally:
             await self.stop_ttfb_metrics()
             # Let the parent class handle TTSStoppedFrame

src/pipecat/services/fal/stt.py

@@ -290,5 +290,5 @@ class FalSTTService(SegmentedSTTService):
                     )
 
         except Exception as e:
-            logger.error(f"Fal Wizper error: {e}")
-            yield ErrorFrame(f"Fal Wizper error: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")

src/pipecat/services/fish/tts.py

@@ -228,7 +228,8 @@ class FishAudioTTSService(InterruptibleTTSService):
 
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"Fish Audio initialization error: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
 
@@ -242,7 +243,8 @@ class FishAudioTTSService(InterruptibleTTSService):
                 await self._websocket.send(ormsgpack.packb(stop_message))
                 await self._websocket.close()
         except Exception as e:
-            logger.error(f"Error closing websocket: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._request_id = None
             self._started = False
@@ -284,7 +286,8 @@ class FishAudioTTSService(InterruptibleTTSService):
                                 continue
 
             except Exception as e:
-                logger.error(f"Error processing message: {e}")
+                logger.error(f"{self} exception: {e}")
+                await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
     @traced_tts
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
@@ -320,7 +323,8 @@ class FishAudioTTSService(InterruptibleTTSService):
                 flush_message = {"event": "flush"}
                 await self._get_websocket().send(ormsgpack.packb(flush_message))
             except Exception as e:
-                logger.error(f"{self} error sending message: {e}")
+                logger.error(f"{self} exception: {e}")
+                yield ErrorFrame(error=f"{self} error: {e}")
                 yield TTSStoppedFrame()
                 await self._disconnect()
                 await self._connect()
@@ -328,5 +332,5 @@ class FishAudioTTSService(InterruptibleTTSService):
             yield None
 
         except Exception as e:
-            logger.error(f"Error generating TTS: {e}")
-            yield ErrorFrame(f"Error: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")

src/pipecat/services/gladia/stt.py

@@ -23,6 +23,7 @@ from pipecat import __version__ as pipecat_version
 from pipecat.frames.frames import (
     CancelFrame,
     EndFrame,
+    ErrorFrame,
     Frame,
     InterimTranscriptionFrame,
     StartFrame,
@@ -467,7 +468,8 @@ class GladiaSTTService(STTService):
                             break
 
             except Exception as e:
-                logger.error(f"Error in connection handler: {e}")
+                logger.error(f"{self} exception: {e}")
+                await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
                 self._connection_active = False
 
                 if not self._should_reconnect:
@@ -557,7 +559,8 @@ class GladiaSTTService(STTService):
         except websockets.exceptions.ConnectionClosed:
             logger.debug("Connection closed during keepalive")
         except Exception as e:
-            logger.error(f"Error in Gladia keepalive task: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
     async def _receive_task_handler(self):
         try:
@@ -620,7 +623,8 @@ class GladiaSTTService(STTService):
             # Expected when closing the connection
             pass
         except Exception as e:
-            logger.error(f"Error in Gladia WebSocket handler: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
     async def _maybe_reconnect(self) -> bool:
         """Handle exponential backoff reconnection logic."""

src/pipecat/services/google/gemini_live/llm.py

@@ -27,6 +27,7 @@ from pydantic import BaseModel, Field
 from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter
 from pipecat.frames.frames import (
+    AggregationType,
     BotStartedSpeakingFrame,
     BotStoppedSpeakingFrame,
     CancelFrame,
@@ -1174,7 +1175,7 @@ class GeminiLiveLLMService(LLMService):
             self._connection_task = self.create_task(self._connection_task_handler(config=config))
 
         except Exception as e:
-            await self.push_error(ErrorFrame(error=f"{self} Initialization error: {e}", fatal=True))
+            await self.push_error(ErrorFrame(error=f"{self} Initialization error: {e}"))
 
     async def _connection_task_handler(self, config: LiveConnectConfig):
         async with self._client.aio.live.connect(model=self._model_name, config=config) as session:
@@ -1255,9 +1256,7 @@ class GeminiLiveLLMService(LLMService):
                 f"Max consecutive failures ({MAX_CONSECUTIVE_FAILURES}) reached, "
                 "treating as fatal error"
             )
-            await self.push_error(
-                ErrorFrame(error=f"{self} Error in receive loop: {error}", fatal=True)
-            )
+            await self.push_error(ErrorFrame(error=f"{self} Error in receive loop: {error}"))
             return False
         else:
             logger.info(
@@ -1453,7 +1452,8 @@ class GeminiLiveLLMService(LLMService):
 
             self._bot_text_buffer += text
             self._search_result_buffer += text  # Also accumulate for grounding
-            await self.push_frame(LLMTextFrame(text=text))
+            frame = LLMTextFrame(text=text)
+            await self.push_frame(frame)
 
         # Check for grounding metadata in server content
         if msg.server_content and msg.server_content.grounding_metadata:
@@ -1645,7 +1645,11 @@ class GeminiLiveLLMService(LLMService):
             await self.push_frame(TTSStartedFrame())
             await self.push_frame(LLMFullResponseStartFrame())
 
-        await self.push_frame(TTSTextFrame(text=text))
+        frame = TTSTextFrame(text=text, aggregated_by=AggregationType.SENTENCE)
+        # Gemini Live text already includes any necessary inter-chunk spaces
+        frame.includes_inter_frame_spaces = True
+
+        await self.push_frame(frame)
 
     async def _handle_msg_grounding_metadata(self, message: LiveServerMessage):
         """Handle dedicated grounding metadata messages."""

src/pipecat/services/google/stt.py

@@ -774,7 +774,8 @@ class GoogleSTTService(STTService):
                 yield cloud_speech.StreamingRecognizeRequest(audio=audio_data)
 
         except Exception as e:
-            logger.error(f"Error in request generator: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             raise
 
     async def _stream_audio(self):
@@ -805,14 +806,15 @@ class GoogleSTTService(STTService):
                         break
 
                 except Exception as e:
-                    logger.warning(f"{self} Reconnecting: {e}")
+                    logger.error(f"{self} exception: {e}")
+                    await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
                     await asyncio.sleep(1)  # Brief delay before reconnecting
                     self._stream_start_time = int(time.time() * 1000)
 
         except Exception as e:
-            logger.error(f"Error in streaming task: {e}")
-            await self.push_frame(ErrorFrame(str(e)))
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
         """Process an audio chunk for STT transcription.
@@ -900,7 +902,8 @@ class GoogleSTTService(STTService):
             )
             raise
         except Exception as e:
-            logger.error(f"Error processing Google STT responses: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             # Re-raise the exception to let it propagate (e.g. in the case of a
             # timeout, propagate to _stream_audio to reconnect)
             raise

src/pipecat/services/google/tts.py

@@ -16,6 +16,7 @@ for natural voice control and multi-speaker conversations.
 
 import json
 import os
+import warnings
 
 from pipecat.utils.tracing.service_decorators import traced_tts
 
@@ -51,19 +52,13 @@ except ModuleNotFoundError as e:
     )
     raise Exception(f"Missing module: {e}")
 
-try:
-    from google import genai
-    from google.genai import types
-
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error("In order to use Gemini TTS, you need to `pip install pipecat-ai[google]`.")
-    raise Exception(f"Missing module: {e}")
-
 
 def language_to_google_tts_language(language: Language) -> Optional[str]:
     """Convert a Language enum to Google TTS language code.
 
+    Source:
+    https://docs.cloud.google.com/text-to-speech/docs/chirp3-hd
+
     Args:
         language: The Language enum value to convert.
 
@@ -71,9 +66,6 @@ def language_to_google_tts_language(language: Language) -> Optional[str]:
         The corresponding Google TTS language code, or None if not supported.
     """
     LANGUAGE_MAP = {
-        # Afrikaans
-        Language.AF: "af-ZA",
-        Language.AF_ZA: "af-ZA",
         # Arabic
         Language.AR: "ar-XA",
         # Bengali
@@ -82,14 +74,9 @@ def language_to_google_tts_language(language: Language) -> Optional[str]:
         # Bulgarian
         Language.BG: "bg-BG",
         Language.BG_BG: "bg-BG",
-        # Catalan
-        Language.CA: "ca-ES",
-        Language.CA_ES: "ca-ES",
-        # Chinese (Mandarin and Cantonese)
-        Language.ZH: "cmn-CN",
-        Language.ZH_CN: "cmn-CN",
-        Language.ZH_TW: "cmn-TW",
-        Language.ZH_HK: "yue-HK",
+        # Croatian
+        Language.HR: "hr-HR",
+        Language.HR_HR: "hr-HR",
         # Czech
         Language.CS: "cs-CZ",
         Language.CS_CZ: "cs-CZ",
@@ -109,9 +96,6 @@ def language_to_google_tts_language(language: Language) -> Optional[str]:
         # Estonian
         Language.ET: "et-EE",
         Language.ET_EE: "et-EE",
-        # Filipino
-        Language.FIL: "fil-PH",
-        Language.FIL_PH: "fil-PH",
         # Finnish
         Language.FI: "fi-FI",
         Language.FI_FI: "fi-FI",
@@ -119,9 +103,6 @@ def language_to_google_tts_language(language: Language) -> Optional[str]:
         Language.FR: "fr-FR",
         Language.FR_CA: "fr-CA",
         Language.FR_FR: "fr-FR",
-        # Galician
-        Language.GL: "gl-ES",
-        Language.GL_ES: "gl-ES",
         # German
         Language.DE: "de-DE",
         Language.DE_DE: "de-DE",
@@ -140,9 +121,6 @@ def language_to_google_tts_language(language: Language) -> Optional[str]:
         # Hungarian
         Language.HU: "hu-HU",
         Language.HU_HU: "hu-HU",
-        # Icelandic
-        Language.IS: "is-IS",
-        Language.IS_IS: "is-IS",
         # Indonesian
         Language.ID: "id-ID",
         Language.ID_ID: "id-ID",
@@ -164,12 +142,12 @@ def language_to_google_tts_language(language: Language) -> Optional[str]:
         # Lithuanian
         Language.LT: "lt-LT",
         Language.LT_LT: "lt-LT",
-        # Malay
-        Language.MS: "ms-MY",
-        Language.MS_MY: "ms-MY",
         # Malayalam
         Language.ML: "ml-IN",
         Language.ML_IN: "ml-IN",
+        # Chinese (Mandarin)
+        Language.ZH: "cmn-CN",
+        Language.ZH_CN: "cmn-CN",
         # Marathi
         Language.MR: "mr-IN",
         Language.MR_IN: "mr-IN",
@@ -181,12 +159,8 @@ def language_to_google_tts_language(language: Language) -> Optional[str]:
         Language.PL: "pl-PL",
         Language.PL_PL: "pl-PL",
         # Portuguese
-        Language.PT: "pt-PT",
+        Language.PT: "pt-BR",
         Language.PT_BR: "pt-BR",
-        Language.PT_PT: "pt-PT",
-        # Punjabi
-        Language.PA: "pa-IN",
-        Language.PA_IN: "pa-IN",
         # Romanian
         Language.RO: "ro-RO",
         Language.RO_RO: "ro-RO",
@@ -199,10 +173,16 @@ def language_to_google_tts_language(language: Language) -> Optional[str]:
         # Slovak
         Language.SK: "sk-SK",
         Language.SK_SK: "sk-SK",
+        # Slovenian
+        Language.SL: "sl-SI",
+        Language.SL_SI: "sl-SI",
         # Spanish
         Language.ES: "es-ES",
         Language.ES_ES: "es-ES",
         Language.ES_US: "es-US",
+        # Swahili
+        Language.SW: "sw-KE",
+        Language.SW_KE: "sw-KE",
         # Swedish
         Language.SV: "sv-SE",
         Language.SV_SE: "sv-SE",
@@ -221,6 +201,9 @@ def language_to_google_tts_language(language: Language) -> Optional[str]:
         # Ukrainian
         Language.UK: "uk-UA",
         Language.UK_UA: "uk-UA",
+        # Urdu
+        Language.UR: "ur-IN",
+        Language.UR_IN: "ur-IN",
         # Vietnamese
         Language.VI: "vi-VN",
         Language.VI_VN: "vi-VN",
@@ -229,6 +212,267 @@ def language_to_google_tts_language(language: Language) -> Optional[str]:
     return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
 
 
+def language_to_gemini_tts_language(language: Language) -> Optional[str]:
+    """Convert a Language enum to Gemini TTS language code.
+
+    Source:
+    https://docs.cloud.google.com/text-to-speech/docs/gemini-tts#available_languages
+
+    Args:
+        language: The Language enum value to convert.
+
+    Returns:
+        The corresponding Gemini TTS language code, or None if not supported.
+    """
+    LANGUAGE_MAP = {
+        # Afrikaans (Preview)
+        Language.AF: "af-ZA",
+        Language.AF_ZA: "af-ZA",
+        # Albanian (Preview)
+        Language.SQ: "sq-AL",
+        Language.SQ_AL: "sq-AL",
+        # Amharic (Preview)
+        Language.AM: "am-ET",
+        Language.AM_ET: "am-ET",
+        # Arabic
+        Language.AR: "ar-EG",  # GA: Egypt
+        Language.AR_EG: "ar-EG",
+        Language.AR_001: "ar-001",  # Preview: World
+        # Armenian (Preview)
+        Language.HY: "hy-AM",
+        Language.HY_AM: "hy-AM",
+        # Azerbaijani (Preview)
+        Language.AZ: "az-AZ",
+        Language.AZ_AZ: "az-AZ",
+        # Basque (Preview)
+        Language.EU: "eu-ES",
+        Language.EU_ES: "eu-ES",
+        # Belarusian (Preview)
+        Language.BE: "be-BY",
+        Language.BE_BY: "be-BY",
+        # Bengali (GA)
+        Language.BN: "bn-BD",
+        Language.BN_BD: "bn-BD",
+        # Bulgarian (Preview)
+        Language.BG: "bg-BG",
+        Language.BG_BG: "bg-BG",
+        # Burmese (Preview)
+        Language.MY: "my-MM",
+        Language.MY_MM: "my-MM",
+        # Catalan (Preview)
+        Language.CA: "ca-ES",
+        Language.CA_ES: "ca-ES",
+        # Cebuano (Preview)
+        Language.CEB: "ceb-PH",
+        Language.CEB_PH: "ceb-PH",
+        # Chinese (Mandarin)
+        Language.ZH: "cmn-CN",  # Preview
+        Language.ZH_CN: "cmn-CN",
+        Language.ZH_TW: "cmn-TW",  # Preview
+        # Croatian (Preview)
+        Language.HR: "hr-HR",
+        Language.HR_HR: "hr-HR",
+        # Czech (Preview)
+        Language.CS: "cs-CZ",
+        Language.CS_CZ: "cs-CZ",
+        # Danish (Preview)
+        Language.DA: "da-DK",
+        Language.DA_DK: "da-DK",
+        # Dutch (GA)
+        Language.NL: "nl-NL",
+        Language.NL_NL: "nl-NL",
+        # English
+        Language.EN: "en-US",  # GA
+        Language.EN_US: "en-US",
+        Language.EN_AU: "en-AU",  # Preview
+        Language.EN_GB: "en-GB",  # Preview
+        Language.EN_IN: "en-IN",  # GA
+        # Estonian (Preview)
+        Language.ET: "et-EE",
+        Language.ET_EE: "et-EE",
+        # Filipino (Preview)
+        Language.FIL: "fil-PH",
+        Language.FIL_PH: "fil-PH",
+        # Finnish (Preview)
+        Language.FI: "fi-FI",
+        Language.FI_FI: "fi-FI",
+        # French
+        Language.FR: "fr-FR",  # GA
+        Language.FR_FR: "fr-FR",
+        Language.FR_CA: "fr-CA",  # Preview
+        # Galician (Preview)
+        Language.GL: "gl-ES",
+        Language.GL_ES: "gl-ES",
+        # Georgian (Preview)
+        Language.KA: "ka-GE",
+        Language.KA_GE: "ka-GE",
+        # German (GA)
+        Language.DE: "de-DE",
+        Language.DE_DE: "de-DE",
+        # Greek (Preview)
+        Language.EL: "el-GR",
+        Language.EL_GR: "el-GR",
+        # Gujarati (Preview)
+        Language.GU: "gu-IN",
+        Language.GU_IN: "gu-IN",
+        # Haitian Creole (Preview)
+        Language.HT: "ht-HT",
+        Language.HT_HT: "ht-HT",
+        # Hebrew (Preview)
+        Language.HE: "he-IL",
+        Language.HE_IL: "he-IL",
+        # Hindi (GA)
+        Language.HI: "hi-IN",
+        Language.HI_IN: "hi-IN",
+        # Hungarian (Preview)
+        Language.HU: "hu-HU",
+        Language.HU_HU: "hu-HU",
+        # Icelandic (Preview)
+        Language.IS: "is-IS",
+        Language.IS_IS: "is-IS",
+        # Indonesian (GA)
+        Language.ID: "id-ID",
+        Language.ID_ID: "id-ID",
+        # Italian (GA)
+        Language.IT: "it-IT",
+        Language.IT_IT: "it-IT",
+        # Japanese (GA)
+        Language.JA: "ja-JP",
+        Language.JA_JP: "ja-JP",
+        # Javanese (Preview)
+        Language.JV: "jv-JV",
+        Language.JV_JV: "jv-JV",
+        # Kannada (Preview)
+        Language.KN: "kn-IN",
+        Language.KN_IN: "kn-IN",
+        # Konkani (Preview)
+        Language.KOK: "kok-IN",
+        Language.KOK_IN: "kok-IN",
+        # Korean (GA)
+        Language.KO: "ko-KR",
+        Language.KO_KR: "ko-KR",
+        # Lao (Preview)
+        Language.LO: "lo-LA",
+        Language.LO_LA: "lo-LA",
+        # Latin (Preview)
+        Language.LA: "la-VA",
+        Language.LA_VA: "la-VA",
+        # Latvian (Preview)
+        Language.LV: "lv-LV",
+        Language.LV_LV: "lv-LV",
+        # Lithuanian (Preview)
+        Language.LT: "lt-LT",
+        Language.LT_LT: "lt-LT",
+        # Luxembourgish (Preview)
+        Language.LB: "lb-LU",
+        Language.LB_LU: "lb-LU",
+        # Macedonian (Preview)
+        Language.MK: "mk-MK",
+        Language.MK_MK: "mk-MK",
+        # Maithili (Preview)
+        Language.MAI: "mai-IN",
+        Language.MAI_IN: "mai-IN",
+        # Malagasy (Preview)
+        Language.MG: "mg-MG",
+        Language.MG_MG: "mg-MG",
+        # Malay (Preview)
+        Language.MS: "ms-MY",
+        Language.MS_MY: "ms-MY",
+        # Malayalam (Preview)
+        Language.ML: "ml-IN",
+        Language.ML_IN: "ml-IN",
+        # Marathi (GA)
+        Language.MR: "mr-IN",
+        Language.MR_IN: "mr-IN",
+        # Mongolian (Preview)
+        Language.MN: "mn-MN",
+        Language.MN_MN: "mn-MN",
+        # Nepali (Preview)
+        Language.NE: "ne-NP",
+        Language.NE_NP: "ne-NP",
+        # Norwegian
+        Language.NO: "nb-NO",  # Preview: Bokmål
+        Language.NB: "nb-NO",
+        Language.NB_NO: "nb-NO",
+        Language.NN: "nn-NO",  # Preview: Nynorsk
+        Language.NN_NO: "nn-NO",
+        # Odia (Preview)
+        Language.OR: "or-IN",
+        Language.OR_IN: "or-IN",
+        # Pashto (Preview)
+        Language.PS: "ps-AF",
+        Language.PS_AF: "ps-AF",
+        # Persian (Preview)
+        Language.FA: "fa-IR",
+        Language.FA_IR: "fa-IR",
+        # Polish (GA)
+        Language.PL: "pl-PL",
+        Language.PL_PL: "pl-PL",
+        # Portuguese
+        Language.PT: "pt-BR",  # GA: Brazil
+        Language.PT_BR: "pt-BR",
+        Language.PT_PT: "pt-PT",  # Preview: Portugal
+        # Punjabi (Preview)
+        Language.PA: "pa-IN",
+        Language.PA_IN: "pa-IN",
+        # Romanian (GA)
+        Language.RO: "ro-RO",
+        Language.RO_RO: "ro-RO",
+        # Russian (GA)
+        Language.RU: "ru-RU",
+        Language.RU_RU: "ru-RU",
+        # Serbian (Preview)
+        Language.SR: "sr-RS",
+        Language.SR_RS: "sr-RS",
+        # Sindhi (Preview)
+        Language.SD: "sd-IN",
+        Language.SD_IN: "sd-IN",
+        # Sinhala (Preview)
+        Language.SI: "si-LK",
+        Language.SI_LK: "si-LK",
+        # Slovak (Preview)
+        Language.SK: "sk-SK",
+        Language.SK_SK: "sk-SK",
+        # Slovenian (Preview)
+        Language.SL: "sl-SI",
+        Language.SL_SI: "sl-SI",
+        # Spanish
+        Language.ES: "es-ES",  # GA
+        Language.ES_ES: "es-ES",
+        Language.ES_419: "es-419",  # Preview: Latin America
+        Language.ES_MX: "es-MX",  # Preview: Mexico
+        # Swahili (Preview)
+        Language.SW: "sw-KE",
+        Language.SW_KE: "sw-KE",
+        # Swedish (Preview)
+        Language.SV: "sv-SE",
+        Language.SV_SE: "sv-SE",
+        # Tamil (GA)
+        Language.TA: "ta-IN",
+        Language.TA_IN: "ta-IN",
+        # Telugu (GA)
+        Language.TE: "te-IN",
+        Language.TE_IN: "te-IN",
+        # Thai (GA)
+        Language.TH: "th-TH",
+        Language.TH_TH: "th-TH",
+        # Turkish (GA)
+        Language.TR: "tr-TR",
+        Language.TR_TR: "tr-TR",
+        # Ukrainian (GA)
+        Language.UK: "uk-UA",
+        Language.UK_UA: "uk-UA",
+        # Urdu (Preview)
+        Language.UR: "ur-PK",
+        Language.UR_PK: "ur-PK",
+        # Vietnamese (GA)
+        Language.VI: "vi-VN",
+        Language.VI_VN: "vi-VN",
+    }
+
+    return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
+
+
 class GoogleHttpTTSService(TTSService):
     """Google Cloud Text-to-Speech HTTP service with SSML support.
 
@@ -493,12 +737,135 @@ class GoogleHttpTTSService(TTSService):
             yield TTSStoppedFrame()
 
         except Exception as e:
-            logger.exception(f"{self} error generating TTS: {e}")
+            logger.error(f"{self} exception: {e}")
             error_message = f"TTS generation error: {str(e)}"
             yield ErrorFrame(error=error_message)
 
 
-class GoogleTTSService(TTSService):
+class GoogleBaseTTSService(TTSService):
+    """Base class for Google Cloud Text-to-Speech streaming services.
+
+    Provides shared streaming synthesis logic for Google TTS services.
+    This is an abstract base class. Use GoogleTTSService or GeminiTTSService instead.
+    """
+
+    def _create_client(
+        self, credentials: Optional[str], credentials_path: Optional[str]
+    ) -> texttospeech_v1.TextToSpeechAsyncClient:
+        """Create authenticated Google Text-to-Speech client.
+
+        Args:
+            credentials: JSON string with service account credentials.
+            credentials_path: Path to service account JSON file.
+
+        Returns:
+            Authenticated TextToSpeechAsyncClient instance.
+
+        Raises:
+            ValueError: If no valid credentials are provided.
+        """
+        creds: Optional[service_account.Credentials] = None
+
+        if credentials:
+            # Use provided credentials JSON string
+            json_account_info = json.loads(credentials)
+            creds = service_account.Credentials.from_service_account_info(json_account_info)
+        elif credentials_path:
+            # Use service account JSON file if provided
+            creds = service_account.Credentials.from_service_account_file(credentials_path)
+        else:
+            try:
+                creds, project_id = default(
+                    scopes=["https://www.googleapis.com/auth/cloud-platform"]
+                )
+            except GoogleAuthError:
+                pass
+
+        if not creds:
+            raise ValueError("No valid credentials provided.")
+
+        return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds)
+
+    def can_generate_metrics(self) -> bool:
+        """Check if this service can generate processing metrics.
+
+        Returns:
+            True, as Google streaming TTS services support metrics generation.
+        """
+        return True
+
+    def language_to_service_language(self, language: Language) -> Optional[str]:
+        """Convert a Language enum to Google TTS language format.
+
+        Args:
+            language: The language to convert.
+
+        Returns:
+            The Google TTS-specific language code, or None if not supported.
+        """
+        return language_to_google_tts_language(language)
+
+    async def _stream_tts(
+        self,
+        streaming_config: texttospeech_v1.StreamingSynthesizeConfig,
+        text: str,
+        prompt: Optional[str] = None,
+    ) -> AsyncGenerator[Frame, None]:
+        """Shared streaming synthesis logic.
+
+        Args:
+            streaming_config: The streaming configuration.
+            text: The text to synthesize.
+            prompt: Optional prompt for style instructions (Gemini only).
+
+        Yields:
+            Frame: Audio frames containing the synthesized speech.
+        """
+        config_request = texttospeech_v1.StreamingSynthesizeRequest(
+            streaming_config=streaming_config
+        )
+
+        async def request_generator():
+            yield config_request
+            synthesis_input_params = {"text": text}
+            if prompt is not None:
+                synthesis_input_params["prompt"] = prompt
+            yield texttospeech_v1.StreamingSynthesizeRequest(
+                input=texttospeech_v1.StreamingSynthesisInput(**synthesis_input_params)
+            )
+
+        streaming_responses = await self._client.streaming_synthesize(request_generator())
+        await self.start_tts_usage_metrics(text)
+
+        yield TTSStartedFrame()
+
+        audio_buffer = b""
+        first_chunk_for_ttfb = False
+
+        CHUNK_SIZE = self.chunk_size
+
+        async for response in streaming_responses:
+            chunk = response.audio_content
+            if not chunk:
+                continue
+
+            if not first_chunk_for_ttfb:
+                await self.stop_ttfb_metrics()
+                first_chunk_for_ttfb = True
+
+            audio_buffer += chunk
+            while len(audio_buffer) >= CHUNK_SIZE:
+                piece = audio_buffer[:CHUNK_SIZE]
+                audio_buffer = audio_buffer[CHUNK_SIZE:]
+                yield TTSAudioRawFrame(piece, self.sample_rate, 1)
+
+        if audio_buffer:
+            yield TTSAudioRawFrame(audio_buffer, self.sample_rate, 1)
+
+        yield TTSStoppedFrame()
+
+
+class GoogleTTSService(GoogleBaseTTSService):
     """Google Cloud Text-to-Speech streaming service.
 
     Provides real-time text-to-speech synthesis using Google Cloud's streaming API
@@ -570,53 +937,6 @@ class GoogleTTSService(TTSService):
             credentials, credentials_path
         )
 
-    def _create_client(
-        self, credentials: Optional[str], credentials_path: Optional[str]
-    ) -> texttospeech_v1.TextToSpeechAsyncClient:
-        creds: Optional[service_account.Credentials] = None
-
-        # Create a Google Cloud service account for the Cloud Text-to-Speech API
-        # Using either the provided credentials JSON string or the path to a service account JSON
-        # file, create a Google Cloud service account and use it to authenticate with the API.
-        if credentials:
-            # Use provided credentials JSON string
-            json_account_info = json.loads(credentials)
-            creds = service_account.Credentials.from_service_account_info(json_account_info)
-        elif credentials_path:
-            # Use service account JSON file if provided
-            creds = service_account.Credentials.from_service_account_file(credentials_path)
-        else:
-            try:
-                creds, project_id = default(
-                    scopes=["https://www.googleapis.com/auth/cloud-platform"]
-                )
-            except GoogleAuthError:
-                pass
-
-        if not creds:
-            raise ValueError("No valid credentials provided.")
-
-        return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds)
-
-    def can_generate_metrics(self) -> bool:
-        """Check if this service can generate processing metrics.
-
-        Returns:
-            True, as Google streaming TTS service supports metrics generation.
-        """
-        return True
-
-    def language_to_service_language(self, language: Language) -> Optional[str]:
-        """Convert a Language enum to Google TTS language format.
-
-        Args:
-            language: The language to convert.
-
-        Returns:
-            The Google TTS-specific language code, or None if not supported.
-        """
-        return language_to_google_tts_language(language)
-
     async def _update_settings(self, settings: Mapping[str, Any]):
         """Override to handle speaking_rate updates for streaming API.
 
@@ -648,6 +968,7 @@ class GoogleTTSService(TTSService):
         try:
             await self.start_ttfb_metrics()
 
+            # Build voice selection params
             if self._voice_cloning_key:
                 voice_clone_params = texttospeech_v1.VoiceCloneParams(
                     voice_cloning_key=self._voice_cloning_key
@@ -660,6 +981,7 @@ class GoogleTTSService(TTSService):
                     language_code=self._settings["language"], name=self._voice_id
                 )
 
+            # Create streaming config
             streaming_config = texttospeech_v1.StreamingSynthesizeConfig(
                 voice=voice,
                 streaming_audio_config=texttospeech_v1.StreamingAudioConfig(
@@ -668,71 +990,40 @@ class GoogleTTSService(TTSService):
                     speaking_rate=self._settings["speaking_rate"],
                 ),
             )
-            config_request = texttospeech_v1.StreamingSynthesizeRequest(
-                streaming_config=streaming_config
-            )
 
-            async def request_generator():
-                yield config_request
-                yield texttospeech_v1.StreamingSynthesizeRequest(
-                    input=texttospeech_v1.StreamingSynthesisInput(text=text)
-                )
-
-            streaming_responses = await self._client.streaming_synthesize(request_generator())
-            await self.start_tts_usage_metrics(text)
-
-            yield TTSStartedFrame()
-
-            audio_buffer = b""
-            first_chunk_for_ttfb = False
-
-            CHUNK_SIZE = self.chunk_size
-
-            async for response in streaming_responses:
-                chunk = response.audio_content
-                if not chunk:
-                    continue
-
-                if not first_chunk_for_ttfb:
-                    await self.stop_ttfb_metrics()
-                    first_chunk_for_ttfb = True
-
-                audio_buffer += chunk
-                while len(audio_buffer) >= CHUNK_SIZE:
-                    piece = audio_buffer[:CHUNK_SIZE]
-                    audio_buffer = audio_buffer[CHUNK_SIZE:]
-                    yield TTSAudioRawFrame(piece, self.sample_rate, 1)
-
-            if audio_buffer:
-                yield TTSAudioRawFrame(audio_buffer, self.sample_rate, 1)
-
-            yield TTSStoppedFrame()
+            # Use base class streaming logic
+            async for frame in self._stream_tts(streaming_config, text):
+                yield frame
 
         except Exception as e:
-            logger.exception(f"{self} error generating TTS: {e}")
+            logger.error(f"{self} exception: {e}")
             error_message = f"TTS generation error: {str(e)}"
             yield ErrorFrame(error=error_message)
 
 
-class GeminiTTSService(TTSService):
-    """Gemini Text-to-Speech service using Gemini TTS models.
+class GeminiTTSService(GoogleBaseTTSService):
+    """Gemini Text-to-Speech streaming service using Gemini TTS models.
 
-    Provides text-to-speech synthesis using Gemini's TTS-specific models
-    (gemini-2.5-flash-preview-tts and gemini-2.5-pro-preview-tts) with
-    support for natural voice control, multiple speakers, and voice styles.
+    Provides real-time text-to-speech synthesis using Gemini's TTS-specific models
+    (gemini-2.5-flash-tts and gemini-2.5-pro-tts) with support for natural
+    voice control, prompts for style instructions, expressive markup tags,
+    and multi-speaker conversations.
 
     Note:
-        Requires Google AI API key. This uses the Gemini API, not Google Cloud TTS.
-        Audio-out is currently a preview feature.
+        Requires Google Cloud credentials via service account JSON, credentials file,
+        or default application credentials (GOOGLE_APPLICATION_CREDENTIALS).
+
+        Uses the Google Cloud Text-to-Speech streaming API for low-latency synthesis.
 
     Example::
 
         tts = GeminiTTSService(
-            api_key="your-google-ai-api-key",
-            model="gemini-2.5-flash-preview-tts",
+            credentials_path="/path/to/service-account.json",
+            model="gemini-2.5-flash-tts",
             voice_id="Kore",
             params=GeminiTTSService.InputParams(
                 language=Language.EN_US,
+                prompt="Say this in a friendly and helpful tone"
             )
         )
     """
@@ -741,36 +1032,36 @@ class GeminiTTSService(TTSService):
 
     # List of available Gemini TTS voices
     AVAILABLE_VOICES = [
-        "Zephyr",
-        "Puck",
+        "Achernar",
+        "Achird",
+        "Algenib",
+        "Algieba",
+        "Alnilam",
+        "Aoede",
+        "Autonoe",
+        "Callirhoe",
         "Charon",
-        "Kore",
+        "Despina",
+        "Enceladus",
+        "Erinome",
         "Fenrir",
+        "Gacrux",
+        "Iapetus",
+        "Kore",
+        "Laomedeia",
         "Leda",
         "Orus",
-        "Aoede",
-        "Callirhoe",
-        "Autonoe",
-        "Enceladus",
-        "Iapetus",
-        "Umbriel",
-        "Algieba",
-        "Despina",
-        "Erinome",
-        "Algenib",
-        "Rasalgethi",
-        "Laomedeia",
-        "Achernar",
-        "Alnilam",
-        "Schedar",
-        "Gacrux",
+        "Puck",
         "Pulcherrima",
-        "Achird",
-        "Zubenelgenubi",
-        "Vindemiatrix",
+        "Rasalgethi",
         "Sadachbia",
         "Sadaltager",
+        "Schedar",
         "Sulafar",
+        "Umbriel",
+        "Vindemiatrix",
+        "Zephyr",
+        "Zubenelgenubi",
     ]
 
     class InputParams(BaseModel):
@@ -778,19 +1069,23 @@ class GeminiTTSService(TTSService):
 
         Parameters:
             language: Language for synthesis. Defaults to English.
+            prompt: Optional style instructions for how to synthesize the content.
             multi_speaker: Whether to enable multi-speaker support.
             speaker_configs: List of speaker configurations for multi-speaker mode.
         """
 
         language: Optional[Language] = Language.EN
+        prompt: Optional[str] = None
         multi_speaker: bool = False
         speaker_configs: Optional[List[dict]] = None
 
     def __init__(
         self,
         *,
-        api_key: str,
-        model: str = "gemini-2.5-flash-preview-tts",
+        api_key: Optional[str] = None,
+        model: str = "gemini-2.5-flash-tts",
+        credentials: Optional[str] = None,
+        credentials_path: Optional[str] = None,
         voice_id: str = "Kore",
         sample_rate: Optional[int] = None,
         params: Optional[InputParams] = None,
@@ -799,14 +1094,30 @@ class GeminiTTSService(TTSService):
         """Initializes the Gemini TTS service.
 
         Args:
-            api_key: Google AI API key for authentication.
+            api_key:
+
+                .. deprecated:: 0.0.95
+                    The `api_key` parameter is deprecated. Use `credentials` or
+                    `credentials_path` instead for Google Cloud authentication.
+
             model: Gemini TTS model to use. Must be a TTS model like
-                   "gemini-2.5-flash-preview-tts" or "gemini-2.5-pro-preview-tts".
+                   "gemini-2.5-flash-tts" or "gemini-2.5-pro-tts".
+            credentials: JSON string containing Google Cloud service account credentials.
+            credentials_path: Path to Google Cloud service account JSON file.
             voice_id: Voice name from the available Gemini voices.
             sample_rate: Audio sample rate in Hz. If None, uses Google's default 24kHz.
             params: TTS configuration parameters.
             **kwargs: Additional arguments passed to parent TTSService.
         """
+        # Handle deprecated api_key parameter
+        if api_key is not None:
+            warnings.warn(
+                "The 'api_key' parameter is deprecated and will be removed in a future version. "
+                "Use 'credentials' or 'credentials_path' instead for Google Cloud authentication.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
         if sample_rate and sample_rate != self.GOOGLE_SAMPLE_RATE:
             logger.warning(
                 f"Google TTS only supports {self.GOOGLE_SAMPLE_RATE}Hz sample rate. "
@@ -819,26 +1130,20 @@ class GeminiTTSService(TTSService):
         if voice_id not in self.AVAILABLE_VOICES:
             logger.warning(f"Voice '{voice_id}' not in known voices list. Using anyway.")
 
-        self._api_key = api_key
         self._model = model
         self._voice_id = voice_id
         self._settings = {
             "language": self.language_to_service_language(params.language)
             if params.language
             else "en-US",
+            "prompt": params.prompt,
             "multi_speaker": params.multi_speaker,
             "speaker_configs": params.speaker_configs,
         }
 
-        self._client = genai.Client(api_key=api_key)
-
-    def can_generate_metrics(self) -> bool:
-        """Check if this service can generate processing metrics.
-
-        Returns:
-            True, as Gemini TTS service supports metrics generation.
-        """
-        return True
+        self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
+            credentials, credentials_path
+        )
 
     def language_to_service_language(self, language: Language) -> Optional[str]:
         """Convert a Language enum to Gemini TTS language format.
@@ -849,7 +1154,7 @@ class GeminiTTSService(TTSService):
         Returns:
             The Gemini TTS-specific language code, or None if not supported.
         """
-        return language_to_google_tts_language(language)
+        return language_to_gemini_tts_language(language)
 
     def set_voice(self, voice_id: str):
         """Set the voice for TTS generation.
@@ -874,90 +1179,75 @@ class GeminiTTSService(TTSService):
                 f"Current rate of {self.sample_rate}Hz may cause issues."
             )
 
-    @traced_tts
-    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        """Generate speech from text using Gemini TTS models.
+    async def _update_settings(self, settings: Mapping[str, Any]):
+        """Override to handle prompt updates.
 
         Args:
-            text: The text to synthesize into speech. Can include natural language
-                  instructions for style, tone, etc.
+            settings: Dictionary of settings to update. Can include 'prompt' (str)
+        """
+        if "prompt" in settings:
+            self._settings["prompt"] = settings["prompt"]
+        await super()._update_settings(settings)
+
+    @traced_tts
+    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+        """Generate streaming speech from text using Gemini TTS models.
+
+        Args:
+            text: The text to synthesize into speech. Can include markup tags
+                  like [sigh], [laughing], [whispering] for expressive control.
 
         Yields:
-            Frame: Audio frames containing the synthesized speech.
+            Frame: Audio frames containing the synthesized speech as it's generated.
         """
         logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
             await self.start_ttfb_metrics()
 
-            # Build the speech config
+            # Build voice selection params
             if self._settings["multi_speaker"] and self._settings["speaker_configs"]:
                 # Multi-speaker mode
                 speaker_voice_configs = []
                 for speaker_config in self._settings["speaker_configs"]:
                     speaker_voice_configs.append(
-                        types.SpeakerVoiceConfig(
-                            speaker=speaker_config["speaker"],
-                            voice_config=types.VoiceConfig(
-                                prebuilt_voice_config=types.PrebuiltVoiceConfig(
-                                    voice_name=speaker_config.get("voice_id", self._voice_id)
-                                )
-                            ),
+                        texttospeech_v1.MultispeakerPrebuiltVoice(
+                            speaker_alias=speaker_config["speaker_alias"],
+                            speaker_id=speaker_config.get("speaker_id", self._voice_id),
                         )
                     )
 
-                speech_config = types.SpeechConfig(
-                    multi_speaker_voice_config=types.MultiSpeakerVoiceConfig(
-                        speaker_voice_configs=speaker_voice_configs
-                    )
+                multi_speaker_voice_config = texttospeech_v1.MultiSpeakerVoiceConfig(
+                    speaker_voice_configs=speaker_voice_configs
+                )
+
+                voice = texttospeech_v1.VoiceSelectionParams(
+                    language_code=self._settings["language"],
+                    model_name=self._model,
+                    multi_speaker_voice_config=multi_speaker_voice_config,
                 )
             else:
                 # Single speaker mode
-                speech_config = types.SpeechConfig(
-                    voice_config=types.VoiceConfig(
-                        prebuilt_voice_config=types.PrebuiltVoiceConfig(voice_name=self._voice_id)
-                    )
+                voice = texttospeech_v1.VoiceSelectionParams(
+                    language_code=self._settings["language"],
+                    name=self._voice_id,
+                    model_name=self._model,
                 )
 
-            # Create the generation config
-            generation_config = types.GenerateContentConfig(
-                response_modalities=["AUDIO"],
-                speech_config=speech_config,
+            # Create streaming config
+            streaming_config = texttospeech_v1.StreamingSynthesizeConfig(
+                voice=voice,
+                streaming_audio_config=texttospeech_v1.StreamingAudioConfig(
+                    audio_encoding=texttospeech_v1.AudioEncoding.PCM,
+                    sample_rate_hertz=self.sample_rate,
+                ),
             )
 
-            # Generate the content
-            response = await self._client.aio.models.generate_content(
-                model=self._model,
-                contents=text,
-                config=generation_config,
-            )
-
-            await self.start_tts_usage_metrics(text)
-
-            yield TTSStartedFrame()
-
-            # Extract audio data from response
-            if response.candidates and len(response.candidates) > 0:
-                candidate = response.candidates[0]
-                if candidate.content and candidate.content.parts:
-                    for part in candidate.content.parts:
-                        if part.inline_data and part.inline_data.mime_type.startswith("audio/"):
-                            audio_data = part.inline_data.data
-                            await self.stop_ttfb_metrics()
-
-                            # Gemini TTS returns PCM audio data, chunk it appropriately
-                            CHUNK_SIZE = self.chunk_size
-
-                            for i in range(0, len(audio_data), CHUNK_SIZE):
-                                chunk = audio_data[i : i + CHUNK_SIZE]
-                                if not chunk:
-                                    break
-                                frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
-                                yield frame
-
-            yield TTSStoppedFrame()
+            # Use base class streaming logic with prompt support
+            async for frame in self._stream_tts(streaming_config, text, self._settings["prompt"]):
+                yield frame
 
         except Exception as e:
-            logger.exception(f"{self} error generating TTS: {e}")
+            logger.error(f"{self} exception: {e}")
             error_message = f"Gemini TTS generation error: {str(e)}"
             yield ErrorFrame(error=error_message)

src/pipecat/services/groq/tts.py

@@ -13,7 +13,13 @@ from typing import AsyncGenerator, Optional
 from loguru import logger
 from pydantic import BaseModel
 
-from pipecat.frames.frames import Frame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame
+from pipecat.frames.frames import (
+    ErrorFrame,
+    Frame,
+    TTSAudioRawFrame,
+    TTSStartedFrame,
+    TTSStoppedFrame,
+)
 from pipecat.services.tts_service import TTSService
 from pipecat.transcriptions.language import Language
 from pipecat.utils.tracing.service_decorators import traced_tts
@@ -141,5 +147,6 @@ class GroqTTSService(TTSService):
                     yield TTSAudioRawFrame(bytes, frame_rate, channels)
         except Exception as e:
             logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
         yield TTSStoppedFrame()

src/pipecat/services/heygen/client.py

@@ -83,6 +83,7 @@ class HeyGenClient:
             version="v2",
         ),
         callbacks: HeyGenCallbacks,
+        connect_as_user: bool = False,
     ) -> None:
         """Initialize the HeyGen client.
 
@@ -92,6 +93,7 @@ class HeyGenClient:
             params: Transport configuration parameters
             session_request: Configuration for the HeyGen session (default: uses Shawn_Therapist_public avatar)
             callbacks: Callback handlers for HeyGen events
+            connect_as_user: Whether to connect using the user token or not (default: False)
         """
         self._api = HeyGenApi(api_key, session=session)
         self._heyGen_session: Optional[HeyGenSession] = None
@@ -119,6 +121,11 @@ class HeyGenClient:
         self._next_send_time = 0
         self._audio_seconds_sent = 0.0
         self._transport_ready = False
+        # HeyGen enforces a protection mechanism that will automatically disconnect the avatar if a user does not join within 5 minutes,
+        # regardless of whether the Pipecat agent remains present in the room.
+        # To prevent unexpected disconnections in HeyGenVideoService, we ensure that a user connection is established using the user's token.
+        # This keeps the avatar session active and avoids forced logouts due to inactivity from the user side.
+        self._connect_as_user = connect_as_user
 
     async def _initialize(self):
         self._heyGen_session = await self._api.new_session(self._session_request)
@@ -562,9 +569,12 @@ class HeyGenClient:
                     self._callbacks.on_participant_disconnected, participant.identity
                 )
 
-            await self._livekit_room.connect(
-                self._heyGen_session.url, self._heyGen_session.livekit_agent_token
+            access_token = (
+                self._heyGen_session.livekit_agent_token
+                if not self._connect_as_user
+                else self._heyGen_session.access_token
             )
+            await self._livekit_room.connect(self._heyGen_session.url, access_token)
             logger.debug(f"Successfully connected to LiveKit room: {self._livekit_room.name}")
             logger.debug(f"Local participant SID: {self._livekit_room.local_participant.sid}")
             logger.debug(

src/pipecat/services/heygen/video.py

@@ -121,6 +121,7 @@ class HeyGenVideoService(AIService):
                 on_participant_connected=self._on_participant_connected,
                 on_participant_disconnected=self._on_participant_disconnected,
             ),
+            connect_as_user=True,
         )
         await self._client.setup(setup)
 

src/pipecat/services/hume/tts.py

@@ -14,12 +14,14 @@ from pydantic import BaseModel
 from pipecat.frames.frames import (
     ErrorFrame,
     Frame,
+    InterruptionFrame,
     StartFrame,
     TTSAudioRawFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
 )
-from pipecat.services.tts_service import TTSService
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.tts_service import WordTTSService
 from pipecat.utils.tracing.service_decorators import traced_tts
 
 try:
@@ -29,6 +31,7 @@ try:
         PostedUtterance,
         PostedUtteranceVoiceWithId,
     )
+    from hume.tts.types import TimestampMessage
 except ModuleNotFoundError as e:  # pragma: no cover - import-time guidance
     logger.error(f"Exception: {e}")
     logger.error("In order to use Hume, you need to `pip install pipecat-ai[hume]`.")
@@ -38,7 +41,7 @@ except ModuleNotFoundError as e:  # pragma: no cover - import-time guidance
 HUME_SAMPLE_RATE = 48_000  # Hume TTS streams at 48 kHz
 
 
-class HumeTTSService(TTSService):
+class HumeTTSService(WordTTSService):
     """Hume Octave Text-to-Speech service.
 
     Streams PCM audio via Hume's HTTP output streaming (JSON chunks) endpoint
@@ -48,6 +51,7 @@ class HumeTTSService(TTSService):
 
     - Generates speech from text using Hume TTS.
     - Streams PCM audio.
+    - Supports word-level timestamps for precise audio-text synchronization.
     - Supports dynamic updates of voice and synthesis parameters at runtime.
     - Provides metrics for Time To First Byte (TTFB) and TTS usage.
     """
@@ -92,7 +96,13 @@ class HumeTTSService(TTSService):
                 f"Hume TTS streams at {HUME_SAMPLE_RATE} Hz; configured sample_rate={sample_rate}"
             )
 
-        super().__init__(sample_rate=sample_rate, **kwargs)
+        # WordTTSService sets push_text_frames=False by default, which we want
+        super().__init__(
+            sample_rate=sample_rate,
+            push_text_frames=False,
+            push_stop_frames=True,
+            **kwargs,
+        )
 
         self._client = AsyncHumeClient(api_key=api_key)
         self._params = params or HumeTTSService.InputParams()
@@ -102,6 +112,10 @@ class HumeTTSService(TTSService):
 
         self._audio_bytes = b""
 
+        # Track cumulative time for word timestamps across utterances
+        self._cumulative_time = 0.0
+        self._started = False
+
     def can_generate_metrics(self) -> bool:
         """Can generate metrics.
 
@@ -117,6 +131,27 @@ class HumeTTSService(TTSService):
             frame: The start frame.
         """
         await super().start(frame)
+        self._reset_state()
+
+    def _reset_state(self):
+        """Reset internal state variables."""
+        self._cumulative_time = 0.0
+        self._started = False
+
+    async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
+        """Push a frame and handle state changes.
+
+        Args:
+            frame: The frame to push.
+            direction: The direction to push the frame.
+        """
+        await super().push_frame(frame, direction)
+        if isinstance(frame, (InterruptionFrame, TTSStoppedFrame)):
+            # Reset timing on interruption or stop
+            self._reset_state()
+
+            if isinstance(frame, TTSStoppedFrame):
+                await self.add_word_timestamps([("Reset", 0)])
 
     async def update_setting(self, key: str, value: Any) -> None:
         """Runtime updates via `TTSUpdateSettingsFrame`.
@@ -133,7 +168,7 @@ class HumeTTSService(TTSService):
 
         if key_l == "voice_id":
             self.set_voice(str(value))
-            logger.info(f"HumeTTSService voice_id set to: {self.voice}")
+            logger.debug(f"HumeTTSService voice_id set to: {self.voice}")
         elif key_l == "description":
             self._params.description = None if value is None else str(value)
         elif key_l == "speed":
@@ -146,7 +181,7 @@ class HumeTTSService(TTSService):
 
     @traced_tts
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        """Generate speech from text using Hume TTS.
+        """Generate speech from text using Hume TTS with word timestamps.
 
         Args:
             text: The text to be synthesized.
@@ -177,7 +212,12 @@ class HumeTTSService(TTSService):
 
         await self.start_ttfb_metrics()
         await self.start_tts_usage_metrics(text)
-        yield TTSStartedFrame()
+
+        # Start TTS sequence if not already started
+        if not self._started:
+            self.start_word_timestamps()
+            yield TTSStartedFrame()
+            self._started = True
 
         try:
             # Instant mode is always enabled here (not user-configurable)
@@ -188,23 +228,50 @@ class HumeTTSService(TTSService):
             # Use version "2" by default if no description is provided
             # Version "1" is needed when description is used
             version = "1" if self._params.description is not None else "2"
+
+            # Track the duration of this utterance based on the last timestamp
+            utterance_duration = 0.0
+
             async for chunk in self._client.tts.synthesize_json_streaming(
                 utterances=[utterance],
                 format=pcm_fmt,
                 instant_mode=True,
                 version=version,
+                include_timestamp_types=["word"],  # Request word-level timestamps
             ):
+                # Process audio chunks
                 audio_b64 = getattr(chunk, "audio", None)
-                if not audio_b64:
-                    continue
+                if audio_b64:
+                    await self.stop_ttfb_metrics()
+                    pcm_bytes = base64.b64decode(audio_b64)
+                    self._audio_bytes += pcm_bytes
 
-                pcm_bytes = base64.b64decode(audio_b64)
-                self._audio_bytes += pcm_bytes
+                    # Buffer audio until we have enough to avoid glitches
+                    if len(self._audio_bytes) >= self.chunk_size:
+                        frame = TTSAudioRawFrame(
+                            audio=self._audio_bytes,
+                            sample_rate=self.sample_rate,
+                            num_channels=1,
+                        )
+                        yield frame
+                        self._audio_bytes = b""
 
-                # Buffer audio until we have enough to avoid glitches
-                if len(self._audio_bytes) < self.chunk_size:
-                    continue
+                # Process timestamp messages
+                if isinstance(chunk, TimestampMessage):
+                    timestamp = chunk.timestamp
+                    if timestamp.type == "word":
+                        # Convert milliseconds to seconds and add cumulative offset
+                        word_start_time = self._cumulative_time + (timestamp.time.begin / 1000.0)
+                        word_end_time = self._cumulative_time + (timestamp.time.end / 1000.0)
 
+                        # Track the maximum end time for this utterance
+                        utterance_duration = max(utterance_duration, word_end_time)
+
+                        # Add word timestamp
+                        await self.add_word_timestamps([(timestamp.text, word_start_time)])
+
+            # Flush any remaining audio bytes
+            if self._audio_bytes:
                 frame = TTSAudioRawFrame(
                     audio=self._audio_bytes,
                     sample_rate=self.sample_rate,
@@ -215,10 +282,14 @@ class HumeTTSService(TTSService):
 
                 self._audio_bytes = b""
 
+            # Update cumulative time for next utterance
+            if utterance_duration > 0:
+                self._cumulative_time = utterance_duration
+
         except Exception as e:
-            logger.exception(f"{self} error generating TTS: {e}")
-            await self.push_error(ErrorFrame(f"Error generating TTS: {e}"))
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             # Ensure TTFB timer is stopped even on early failures
             await self.stop_ttfb_metrics()
-            yield TTSStoppedFrame()
+            # Let the parent class handle TTSStoppedFrame via push_stop_frames

src/pipecat/services/inworld/tts.py

@@ -146,6 +146,8 @@ class InworldTTSService(TTSService):
         Parameters:
             temperature: Voice temperature control for synthesis variability (e.g., 1.1).
                         Valid range: [0, 2]. Higher values increase variability.
+            speaking_rate: Speaking speed control (range: [0.5, 1.5]). Defaults to 1.0 when
+                           unset.
 
         Note:
             Language is automatically inferred from the input text by Inworld's TTS models,
@@ -153,6 +155,7 @@ class InworldTTSService(TTSService):
         """
 
         temperature: Optional[float] = None  # optional temperature control (range: [0, 2])
+        speaking_rate: Optional[float] = None  # optional speaking rate control (range: [0.5, 1.5])
 
     def __init__(
         self,
@@ -198,6 +201,7 @@ class InworldTTSService(TTSService):
                      - Other formats as supported by Inworld API
             params: Optional input parameters for additional configuration. Use this to specify:
                    - temperature: Voice temperature control for variability (range: [0, 2], e.g., 1.1, optional)
+                   - speaking_rate: Set desired speaking speed (range: [0.5, 1.5], optional)
                    Language is automatically inferred from input text.
             **kwargs: Additional arguments passed to the parent TTSService class.
 
@@ -228,15 +232,18 @@ class InworldTTSService(TTSService):
         self._settings = {
             "voiceId": voice_id,  # Voice selection from direct parameter
             "modelId": model,  # TTS model selection from direct parameter
-            "audio_config": {  # Audio format configuration
-                "audio_encoding": encoding,  # Format: LINEAR16, MP3, etc.
-                "sample_rate_hertz": 0,  # Will be set in start() from parent service
+            "audioConfig": {  # Audio format configuration
+                "audioEncoding": encoding,  # Format: LINEAR16, MP3, etc.
+                "sampleRateHertz": 0,  # Will be set in start() from parent service
             },
         }
 
         # Add optional temperature parameter if provided (valid range: [0, 2])
         if params and params.temperature is not None:
             self._settings["temperature"] = params.temperature
+        # Add optional speaking rate if provided (valid range: [0.5, 1.5])
+        if params and params.speaking_rate is not None:
+            self._settings["audioConfig"]["speakingRate"] = params.speaking_rate
 
         # Register voice and model with parent service for metrics and tracking
         self.set_voice(voice_id)  # Used for logging and metrics
@@ -257,7 +264,7 @@ class InworldTTSService(TTSService):
             frame: The start frame containing initialization parameters.
         """
         await super().start(frame)
-        self._settings["audio_config"]["sample_rate_hertz"] = self.sample_rate
+        self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate
 
     async def stop(self, frame: EndFrame):
         """Stop the Inworld TTS service.
@@ -323,9 +330,7 @@ class InworldTTSService(TTSService):
             "text": text,  # Text to synthesize
             "voiceId": self._settings["voiceId"],  # Voice selection (Ashley, Hades, etc.)
             "modelId": self._settings["modelId"],  # TTS model (inworld-tts-1)
-            "audio_config": self._settings[
-                "audio_config"
-            ],  # Audio format settings (LINEAR16, 48kHz)
+            "audioConfig": self._settings["audioConfig"],  # Audio format settings (LINEAR16, 48kHz)
         }
 
         # Add optional temperature parameter if configured (valid range: [0, 2])
@@ -365,7 +370,7 @@ class InworldTTSService(TTSService):
                 if response.status != 200:
                     error_text = await response.text()
                     logger.error(f"Inworld API error: {error_text}")
-                    await self.push_error(ErrorFrame(f"Inworld API error: {error_text}"))
+                    yield ErrorFrame(error=f"Inworld API error: {error_text}")
                     return
 
                 # ================================================================================
@@ -393,7 +398,7 @@ class InworldTTSService(TTSService):
             # ================================================================================
             # Log any unexpected errors and notify the pipeline
             logger.error(f"{self} exception: {e}")
-            await self.push_error(ErrorFrame(f"Error generating TTS: {e}"))
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             # ================================================================================
             # STEP 8: CLEANUP AND COMPLETION
@@ -508,7 +513,7 @@ class InworldTTSService(TTSService):
         # Extract the base64-encoded audio content from response
         if "audioContent" not in response_data:
             logger.error("No audioContent in Inworld API response")
-            await self.push_error(ErrorFrame("No audioContent in response"))
+            await self.push_error(ErrorFrame(error="No audioContent in response"))
             return
 
         # ================================================================================

src/pipecat/services/lmnt/tts.py

@@ -214,7 +214,8 @@ class LmntTTSService(InterruptibleTTSService):
 
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"{self} initialization error: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
 
@@ -230,7 +231,8 @@ class LmntTTSService(InterruptibleTTSService):
                 # await self._websocket.send(json.dumps({"eof": True}))
                 await self._websocket.close()
         except Exception as e:
-            logger.error(f"{self} error closing websocket: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._started = False
             self._websocket = None
@@ -267,7 +269,7 @@ class LmntTTSService(InterruptibleTTSService):
                         logger.error(f"{self} error: {msg['error']}")
                         await self.push_frame(TTSStoppedFrame())
                         await self.stop_all_metrics()
-                        await self.push_error(ErrorFrame(f"{self} error: {msg['error']}"))
+                        await self.push_error(ErrorFrame(error=f"{self} error: {msg['error']}"))
                         return
                 except json.JSONDecodeError:
                     logger.error(f"Invalid JSON message: {message}")
@@ -300,7 +302,8 @@ class LmntTTSService(InterruptibleTTSService):
                 await self._get_websocket().send(json.dumps({"flush": True}))
                 await self.start_tts_usage_metrics(text)
             except Exception as e:
-                logger.error(f"{self} error sending message: {e}")
+                logger.error(f"{self} exception: {e}")
+                yield ErrorFrame(error=f"{self} error: {e}")
                 yield TTSStoppedFrame()
                 await self._disconnect()
                 await self._connect()
@@ -308,3 +311,4 @@ class LmntTTSService(InterruptibleTTSService):
             yield None
         except Exception as e:
             logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")

src/pipecat/services/minimax/tts.py

@@ -40,24 +40,40 @@ def language_to_minimax_language(language: Language) -> Optional[str]:
         The corresponding MiniMax language name, or None if not supported.
     """
     LANGUAGE_MAP = {
+        Language.AF: "Afrikaans",
         Language.AR: "Arabic",
+        Language.BG: "Bulgarian",
+        Language.CA: "Catalan",
         Language.CS: "Czech",
+        Language.DA: "Danish",
         Language.DE: "German",
         Language.EL: "Greek",
         Language.EN: "English",
         Language.ES: "Spanish",
+        Language.FA: "Persian",  # ⚠️ Only supported by speech-2.6-* models
         Language.FI: "Finnish",
+        Language.FIL: "Filipino",  # ⚠️ Only supported by speech-2.6-* models
         Language.FR: "French",
+        Language.HE: "Hebrew",
         Language.HI: "Hindi",
+        Language.HR: "Croatian",
+        Language.HU: "Hungarian",
         Language.ID: "Indonesian",
         Language.IT: "Italian",
         Language.JA: "Japanese",
         Language.KO: "Korean",
+        Language.MS: "Malay",
+        Language.NB: "Norwegian",
+        Language.NN: "Nynorsk",
         Language.NL: "Dutch",
         Language.PL: "Polish",
         Language.PT: "Portuguese",
         Language.RO: "Romanian",
         Language.RU: "Russian",
+        Language.SK: "Slovak",
+        Language.SL: "Slovenian",
+        Language.SV: "Swedish",
+        Language.TA: "Tamil",  # ⚠️ Only supported by speech-2.6-* models
         Language.TH: "Thai",
         Language.TR: "Turkish",
         Language.UK: "Ukrainian",
@@ -84,13 +100,22 @@ class MiniMaxHttpTTSService(TTSService):
         """Configuration parameters for MiniMax TTS.
 
         Parameters:
-            language: Language for TTS generation.
+            language: Language for TTS generation. Supports 40 languages.
+                Note: Filipino, Tamil, and Persian require speech-2.6-* models.
             speed: Speech speed (range: 0.5 to 2.0).
             volume: Speech volume (range: 0 to 10).
             pitch: Pitch adjustment (range: -12 to 12).
             emotion: Emotional tone (options: "happy", "sad", "angry", "fearful",
-                "disgusted", "surprised", "neutral").
-            english_normalization: Whether to apply English text normalization.
+                "disgusted", "surprised", "calm", "fluent").
+            english_normalization: Deprecated; use `text_normalization` instead
+
+                .. deprecated:: 0.0.96
+                    The `english_normalization` parameter is deprecated and will be removed in a future version.
+                    Use the `text_normalization` parameter instead.
+
+            text_normalization: Enable text normalization (Chinese/English).
+            latex_read: Enable LaTeX formula reading.
+            exclude_aggregated_audio: Whether to exclude aggregated audio in final chunk.
         """
 
         language: Optional[Language] = Language.EN
@@ -98,7 +123,10 @@ class MiniMaxHttpTTSService(TTSService):
         volume: Optional[float] = 1.0
         pitch: Optional[int] = 0
         emotion: Optional[str] = None
-        english_normalization: Optional[bool] = None
+        english_normalization: Optional[bool] = None  # Deprecated
+        text_normalization: Optional[bool] = None
+        latex_read: Optional[bool] = None
+        exclude_aggregated_audio: Optional[bool] = None
 
     def __init__(
         self,
@@ -120,9 +148,12 @@ class MiniMaxHttpTTSService(TTSService):
             base_url: API base URL, defaults to MiniMax's T2A endpoint.
                 Global: https://api.minimax.io/v1/t2a_v2
                 Mainland China: https://api.minimaxi.chat/v1/t2a_v2
+                Western United States: https://api-uw.minimax.io/v1/t2a_v2
             group_id: MiniMax Group ID to identify project.
-            model: TTS model name. Defaults to "speech-02-turbo". Options include
-                "speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo".
+            model: TTS model name. Defaults to "speech-02-turbo". Options include:
+                "speech-2.6-hd", "speech-2.6-turbo" (latest, supports Filipino/Tamil/Persian),
+                "speech-02-hd", "speech-02-turbo",
+                "speech-01-hd", "speech-01-turbo".
             voice_id: Voice identifier. Defaults to "Calm_Woman".
             aiohttp_session: aiohttp.ClientSession for API communication.
             sample_rate: Output audio sample rate in Hz. If None, uses pipeline default.
@@ -176,15 +207,34 @@ class MiniMaxHttpTTSService(TTSService):
                 "disgusted",
                 "surprised",
                 "neutral",
+                "fluent",
             ]
             if params.emotion in supported_emotions:
                 self._settings["voice_setting"]["emotion"] = params.emotion
             else:
-                logger.warning(f"Unsupported emotion: {params.emotion}. Using default.")
+                logger.warning(
+                    f"Unsupported emotion: {params.emotion}. Supported emotions: {supported_emotions}"
+                )
 
-        # Add english_normalization if provided
+        # If `english_normalization`, add `text_normalization` and print warning
         if params.english_normalization is not None:
-            self._settings["english_normalization"] = params.english_normalization
+            import warnings
+
+            with warnings.catch_warnings():
+                warnings.simplefilter("always")
+                warnings.warn(
+                    "Parameter `english_normalization` is deprecated and will be removed in a future version. Use `text_normalization` instead.",
+                    DeprecationWarning,
+                )
+            self._settings["voice_setting"]["text_normalization"] = params.english_normalization
+
+        # Add text_normalization if provided (corrected parameter name)
+        if params.text_normalization is not None:
+            self._settings["voice_setting"]["text_normalization"] = params.text_normalization
+
+        # Add latex_read if provided
+        if params.latex_read is not None:
+            self._settings["voice_setting"]["latex_read"] = params.latex_read
 
     def can_generate_metrics(self) -> bool:
         """Check if this service can generate processing metrics.
@@ -231,7 +281,7 @@ class MiniMaxHttpTTSService(TTSService):
         """
         await super().start(frame)
         self._settings["audio_setting"]["sample_rate"] = self.sample_rate
-        logger.debug(f"MiniMax TTS initialized with sample rate: {self.sample_rate}")
+        logger.debug(f"MiniMax TTS initialized with sample_rate: {self.sample_rate}")
 
     @traced_tts
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
@@ -330,16 +380,20 @@ class MiniMaxHttpTTSService(TTSService):
                                             num_channels=1,
                                         )
                                 except ValueError as e:
-                                    logger.error(f"Error converting hex to binary: {e}")
+                                    logger.error(
+                                        f"Error converting hex to binary: {e}",
+                                    )
                                     continue
 
                         except json.JSONDecodeError as e:
-                            logger.error(f"Error decoding JSON: {e}, data: {data_block[:100]}")
+                            logger.error(
+                                f"Error decoding JSON: {e}, data: {data_block[:100]}",
+                            )
                             continue
 
         except Exception as e:
-            logger.exception(f"Error generating TTS: {e}")
-            yield ErrorFrame(error=f"MiniMax TTS error: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
         finally:
             await self.stop_ttfb_metrics()
             yield TTSStoppedFrame()

src/pipecat/services/neuphonic/tts.py

@@ -117,7 +117,6 @@ class NeuphonicTTSService(InterruptibleTTSService):
         """
         super().__init__(
             aggregate_sentences=aggregate_sentences,
-            push_text_frames=False,
             push_stop_frames=True,
             stop_frame_timeout_s=2.0,
             sample_rate=sample_rate,
@@ -286,7 +285,8 @@ class NeuphonicTTSService(InterruptibleTTSService):
 
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"{self} initialization error: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
 
@@ -299,7 +299,8 @@ class NeuphonicTTSService(InterruptibleTTSService):
                 logger.debug("Disconnecting from Neuphonic")
                 await self._websocket.close()
         except Exception as e:
-            logger.error(f"{self} error closing websocket: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._started = False
             self._websocket = None
@@ -364,7 +365,8 @@ class NeuphonicTTSService(InterruptibleTTSService):
                 await self._send_text(text)
                 await self.start_tts_usage_metrics(text)
             except Exception as e:
-                logger.error(f"{self} error sending message: {e}")
+                logger.error(f"{self} exception: {e}")
+                yield ErrorFrame(error=f"{self} error: {e}")
                 yield TTSStoppedFrame()
                 await self._disconnect()
                 await self._connect()
@@ -372,6 +374,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
             yield None
         except Exception as e:
             logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
 
 class NeuphonicHttpTTSService(TTSService):
@@ -565,7 +568,8 @@ class NeuphonicHttpTTSService(TTSService):
                             yield TTSAudioRawFrame(audio_bytes, self.sample_rate, 1)
 
                     except Exception as e:
-                        logger.error(f"Error processing SSE message: {e}")
+                        logger.error(f"{self} exception: {e}")
+                        yield ErrorFrame(error=f"{self} error: {e}")
                         # Don't yield error frame for individual message failures
                         continue
 
@@ -573,8 +577,8 @@ class NeuphonicHttpTTSService(TTSService):
             logger.debug("TTS generation cancelled")
             raise
         except Exception as e:
-            logger.exception(f"Error in run_tts: {e}")
-            yield ErrorFrame(error=f"Neuphonic TTS error: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
         finally:
             await self.stop_ttfb_metrics()
             yield TTSStoppedFrame()

src/pipecat/services/openai/realtime/llm.py

@@ -19,6 +19,7 @@ from pipecat.adapters.services.open_ai_realtime_adapter import (
     OpenAIRealtimeLLMAdapter,
 )
 from pipecat.frames.frames import (
+    AggregationType,
     BotStoppedSpeakingFrame,
     CancelFrame,
     EndFrame,
@@ -477,7 +478,7 @@ class OpenAIRealtimeLLMService(LLMService):
             # it is to recover from a send-side error with proper state management, and that exponential
             # backoff for retries can have cost/stability implications for a service cluster, let's just
             # treat a send-side error as fatal.
-            await self.push_error(ErrorFrame(error=f"Error sending client event: {e}", fatal=True))
+            await self.push_error(ErrorFrame(error=f"Error sending client event: {e}"))
 
     async def _update_settings(self):
         settings = self._session_properties
@@ -673,9 +674,7 @@ class OpenAIRealtimeLLMService(LLMService):
         self._current_assistant_response = None
         # error handling
         if evt.response.status == "failed":
-            await self.push_error(
-                ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True)
-            )
+            await self.push_error(ErrorFrame(error=evt.response.status_details["error"]["message"]))
             return
         # response content
         for item in evt.response.output:
@@ -685,13 +684,17 @@ class OpenAIRealtimeLLMService(LLMService):
         # We receive text deltas (as opposed to audio transcript deltas) when
         # the output modality is "text"
         if evt.delta:
-            await self.push_frame(LLMTextFrame(evt.delta))
+            frame = LLMTextFrame(evt.delta)
+            await self.push_frame(frame)
 
     async def _handle_evt_audio_transcript_delta(self, evt):
         # We receive audio transcript deltas (as opposed to text deltas) when
         # the output modality is "audio" (the default)
         if evt.delta:
-            await self.push_frame(TTSTextFrame(evt.delta))
+            frame = TTSTextFrame(evt.delta, aggregated_by=AggregationType.SENTENCE)
+            # OpenAI Realtime text already includes any necessary inter-chunk spaces
+            frame.includes_inter_frame_spaces = True
+            await self.push_frame(frame)
 
     async def _handle_evt_function_call_arguments_done(self, evt):
         """Handle completion of function call arguments.
@@ -763,7 +766,7 @@ class OpenAIRealtimeLLMService(LLMService):
 
     async def _handle_evt_error(self, evt):
         # Errors are fatal to this connection. Send an ErrorFrame.
-        await self.push_error(ErrorFrame(error=f"Error: {evt}", fatal=True))
+        await self.push_error(ErrorFrame(error=f"Error: {evt}"))
 
     #
     # state and client events for the current conversation

src/pipecat/services/openai/tts.py

@@ -190,7 +190,7 @@ class OpenAITTSService(TTSService):
                         f"{self} error getting audio (status: {r.status_code}, error: {error})"
                     )
                     yield ErrorFrame(
-                        f"Error getting audio (status: {r.status_code}, error: {error})"
+                        error=f"Error getting audio (status: {r.status_code}, error: {error})"
                     )
                     return
 
@@ -207,3 +207,4 @@ class OpenAITTSService(TTSService):
                 yield TTSStoppedFrame()
         except BadRequestError as e:
             logger.exception(f"{self} error generating TTS: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")

src/pipecat/services/openai_realtime_beta/openai.py

@@ -17,6 +17,7 @@ from loguru import logger
 
 from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter
 from pipecat.frames.frames import (
+    AggregationType,
     BotStoppedSpeakingFrame,
     CancelFrame,
     EndFrame,
@@ -454,7 +455,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             # it is to recover from a send-side error with proper state management, and that exponential
             # backoff for retries can have cost/stability implications for a service cluster, let's just
             # treat a send-side error as fatal.
-            await self.push_error(ErrorFrame(error=f"Error sending client event: {e}", fatal=True))
+            await self.push_error(ErrorFrame(error=f"Error sending client event: {e}"))
 
     async def _update_settings(self):
         settings = self._session_properties
@@ -627,9 +628,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         self._current_assistant_response = None
         # error handling
         if evt.response.status == "failed":
-            await self.push_error(
-                ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True)
-            )
+            await self.push_error(ErrorFrame(error=evt.response.status_details["error"]["message"]))
             return
         # response content
         for item in evt.response.output:
@@ -654,7 +653,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
     async def _handle_evt_audio_transcript_delta(self, evt):
         if evt.delta:
             await self.push_frame(LLMTextFrame(evt.delta))
-            await self.push_frame(TTSTextFrame(evt.delta))
+            await self.push_frame(TTSTextFrame(evt.delta, aggregated_by=AggregationType.SENTENCE))
 
     async def _handle_evt_speech_started(self, evt):
         await self._truncate_current_audio_response()
@@ -687,7 +686,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
 
     async def _handle_evt_error(self, evt):
         # Errors are fatal to this connection. Send an ErrorFrame.
-        await self.push_error(ErrorFrame(error=f"Error: {evt}", fatal=True))
+        await self.push_error(ErrorFrame(error=f"Error: {evt}"))
 
     async def _handle_assistant_output(self, output):
         # We haven't seen intermixed audio and function_call items in the same response. But let's

src/pipecat/services/piper/tts.py

@@ -92,7 +92,7 @@ class PiperTTSService(TTSService):
                         f"{self} error getting audio (status: {response.status}, error: {error})"
                     )
                     yield ErrorFrame(
-                        f"Error getting audio (status: {response.status}, error: {error})"
+                        error=f"Error getting audio (status: {response.status}, error: {error})"
                     )
                     return
 
@@ -108,8 +108,8 @@ class PiperTTSService(TTSService):
                     await self.stop_ttfb_metrics()
                     yield frame
         except Exception as e:
-            logger.error(f"Error in run_tts: {e}")
-            yield ErrorFrame(error=str(e))
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
         finally:
             logger.debug(f"{self}: Finished TTS [{text}]")
             await self.stop_ttfb_metrics()

src/pipecat/services/playht/tts.py

@@ -266,7 +266,8 @@ class PlayHTTTSService(InterruptibleTTSService):
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
         except Exception as e:
-            logger.error(f"{self} initialization error: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
 
@@ -279,7 +280,8 @@ class PlayHTTTSService(InterruptibleTTSService):
                 logger.debug("Disconnecting from PlayHT")
                 await self._websocket.close()
         except Exception as e:
-            logger.error(f"{self} error closing websocket: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._request_id = None
             self._websocket = None
@@ -350,7 +352,7 @@ class PlayHTTTSService(InterruptibleTTSService):
                             self._request_id = None
                     elif "error" in msg:
                         logger.error(f"{self} error: {msg}")
-                        await self.push_error(ErrorFrame(f"{self} error: {msg['error']}"))
+                        await self.push_error(ErrorFrame(error=f"{self} error: {msg['error']}"))
                 except json.JSONDecodeError:
                     logger.error(f"Invalid JSON message: {message}")
 
@@ -392,7 +394,8 @@ class PlayHTTTSService(InterruptibleTTSService):
                 await self._get_websocket().send(json.dumps(tts_command))
                 await self.start_tts_usage_metrics(text)
             except Exception as e:
-                logger.error(f"{self} error sending message: {e}")
+                logger.error(f"{self} exception: {e}")
+                yield ErrorFrame(error=f"{self} error: {e}")
                 yield TTSStoppedFrame()
                 await self._disconnect()
                 await self._connect()
@@ -402,8 +405,8 @@ class PlayHTTTSService(InterruptibleTTSService):
             yield None
 
         except Exception as e:
-            logger.error(f"{self} error generating TTS: {e}")
-            yield ErrorFrame(f"{self} error: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
 
 class PlayHTHttpTTSService(TTSService):
@@ -623,7 +626,8 @@ class PlayHTHttpTTSService(TTSService):
                             yield frame
 
         except Exception as e:
-            logger.error(f"{self} error generating TTS: {e}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
         finally:
             await self.stop_ttfb_metrics()
             yield TTSStoppedFrame()

src/pipecat/services/rime/tts.py

@@ -60,6 +60,7 @@ def language_to_rime_language(language: Language) -> str:
         Language.FR: "fra",
         Language.EN: "eng",
         Language.ES: "spa",
+        Language.HI: "hin",
     }
     return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
 
@@ -112,6 +113,10 @@ class RimeTTSService(AudioContextWordTTSService):
             sample_rate: Audio sample rate in Hz.
             params: Additional configuration parameters.
             text_aggregator: Custom text aggregator for processing input text.
+
+                .. deprecated:: 0.0.95
+                    Use an LLMTextProcessor before the TTSService for custom text aggregation.
+
             aggregate_sentences: Whether to aggregate sentences within the TTSService.
             **kwargs: Additional arguments passed to parent class.
         """
@@ -122,10 +127,17 @@ class RimeTTSService(AudioContextWordTTSService):
             push_stop_frames=True,
             pause_frame_processing=True,
             sample_rate=sample_rate,
-            text_aggregator=text_aggregator or SkipTagsAggregator([("spell(", ")")]),
             **kwargs,
         )
 
+        if not text_aggregator:
+            # Always skip tags added for spelled-out text
+            # Note: This is primarily to support backwards compatibility.
+            #    The preferred way of taking advantage of Rime spelling is
+            #    to use an LLMTextProcessor and/or a text_transformer to identify
+            #    and insert these tags for the purpose of the TTS service alone.
+            self._text_aggregator = SkipTagsAggregator([("spell(", ")")])
+
         params = params or RimeTTSService.InputParams()
 
         # Store service configuration
@@ -151,6 +163,7 @@ class RimeTTSService(AudioContextWordTTSService):
         self._context_id = None  # Tracks current turn
         self._receive_task = None
         self._cumulative_time = 0  # Accumulates time across messages
+        self._extra_msg_fields = {}  # Extra fields for next message
 
     def can_generate_metrics(self) -> bool:
         """Check if this service can generate processing metrics.
@@ -180,6 +193,31 @@ class RimeTTSService(AudioContextWordTTSService):
         self._model = model
         await super().set_model(model)
 
+    # A set of Rime-specific helpers for text transformations
+    def SPELL(text: str) -> str:
+        """Wrap text in Rime spell function."""
+        return f"spell({text})"
+
+    def PAUSE_TAG(seconds: float) -> str:
+        """Convenience method to create a pause tag."""
+        return f"<{seconds * 1000}>"
+
+    def PRONOUNCE(self, text: str, word: str, phoneme: str) -> str:
+        """Convenience method to support Rime's custom pronunciations feature.
+
+        https://docs.rime.ai/api-reference/custom-pronunciation
+        """
+        self._extra_msg_fields["phonemizeBetweenBrackets"] = True
+        return text.replace(word, f"{phoneme}")
+
+    def INLINE_SPEED(self, text: str, speed: float) -> str:
+        """Convenience method to support inline speeds."""
+        if not self._extra_msg_fields:
+            self._extra_msg_fields = {}
+        speed_vals = self._extra_msg_fields.get("inlineSpeedAlpha", "").split(",")
+        self._extra_msg_fields["inlineSpeedAlpha"] = ",".join(speed_vals + [str(speed)])
+        return f"[{text}]"
+
     async def _update_settings(self, settings: Mapping[str, Any]):
         """Update service settings and reconnect if voice changed."""
         prev_voice = self._voice_id
@@ -192,7 +230,11 @@ class RimeTTSService(AudioContextWordTTSService):
 
     def _build_msg(self, text: str = "") -> dict:
         """Build JSON message for Rime API."""
-        return {"text": text, "contextId": self._context_id}
+        msg = {"text": text, "contextId": self._context_id}
+        if self._extra_msg_fields:
+            msg |= self._extra_msg_fields
+            self._extra_msg_fields = {}
+        return msg
 
     def _build_clear_msg(self) -> dict:
         """Build clear operation message."""
@@ -258,7 +300,8 @@ class RimeTTSService(AudioContextWordTTSService):
 
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"{self} initialization error: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
 
@@ -270,7 +313,8 @@ class RimeTTSService(AudioContextWordTTSService):
                 await self._websocket.send(json.dumps(self._build_eos_msg()))
                 await self._websocket.close()
         except Exception as e:
-            logger.error(f"{self} error closing websocket: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._context_id = None
             self._websocket = None
@@ -366,7 +410,7 @@ class RimeTTSService(AudioContextWordTTSService):
                 logger.error(f"{self} error: {msg}")
                 await self.push_frame(TTSStoppedFrame())
                 await self.stop_all_metrics()
-                await self.push_error(ErrorFrame(f"{self} error: {msg['message']}"))
+                await self.push_error(ErrorFrame(error=f"{self} error: {msg['message']}"))
                 self._context_id = None
 
     async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
@@ -408,7 +452,8 @@ class RimeTTSService(AudioContextWordTTSService):
                 await self._get_websocket().send(json.dumps(msg))
                 await self.start_tts_usage_metrics(text)
             except Exception as e:
-                logger.error(f"{self} error sending message: {e}")
+                logger.error(f"{self} exception: {e}")
+                yield ErrorFrame(error=f"{self} error: {e}")
                 yield TTSStoppedFrame()
                 await self._disconnect()
                 await self._connect()
@@ -416,6 +461,7 @@ class RimeTTSService(AudioContextWordTTSService):
             yield None
         except Exception as e:
             logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
 
 class RimeHttpTTSService(TTSService):
@@ -564,8 +610,8 @@ class RimeHttpTTSService(TTSService):
                     yield frame
 
         except Exception as e:
-            logger.exception(f"Error generating TTS: {e}")
-            yield ErrorFrame(error=f"Rime TTS error: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
         finally:
             await self.stop_ttfb_metrics()
             yield TTSStoppedFrame()

src/pipecat/services/riva/stt.py

@@ -659,8 +659,8 @@ class RivaSegmentedSTTService(SegmentedSTTService):
                 yield ErrorFrame(f"Unexpected Riva response format: {str(ae)}")
 
         except Exception as e:
-            logger.exception(f"Riva Canary ASR error: {e}")
-            yield ErrorFrame(f"Riva Canary ASR error: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
 
 class ParakeetSTTService(RivaSTTService):

src/pipecat/services/riva/tts.py

@@ -23,6 +23,7 @@ from loguru import logger
 from pydantic import BaseModel
 
 from pipecat.frames.frames import (
+    ErrorFrame,
     Frame,
     TTSAudioRawFrame,
     TTSStartedFrame,
@@ -180,6 +181,7 @@ class RivaTTSService(TTSService):
                 resp = await asyncio.wait_for(queue.get(), timeout=RIVA_TTS_TIMEOUT_SECS)
         except asyncio.TimeoutError:
             logger.error(f"{self} timeout waiting for audio response")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
         await self.start_tts_usage_metrics(text)
         yield TTSStoppedFrame()

src/pipecat/services/sarvam/tts.py

@@ -255,7 +255,7 @@ class SarvamHttpTTSService(TTSService):
                 if response.status != 200:
                     error_text = await response.text()
                     logger.error(f"Sarvam API error: {error_text}")
-                    await self.push_error(ErrorFrame(f"Sarvam API error: {error_text}"))
+                    await self.push_error(ErrorFrame(error=f"Sarvam API error: {error_text}"))
                     return
 
                 response_data = await response.json()
@@ -265,7 +265,7 @@ class SarvamHttpTTSService(TTSService):
             # Decode base64 audio data
             if "audios" not in response_data or not response_data["audios"]:
                 logger.error("No audio data received from Sarvam API")
-                await self.push_error(ErrorFrame("No audio data received"))
+                await self.push_error(ErrorFrame(error="No audio data received"))
                 return
 
             # Get the first audio (there should be only one for single text input)
@@ -287,7 +287,7 @@ class SarvamHttpTTSService(TTSService):
 
         except Exception as e:
             logger.error(f"{self} exception: {e}")
-            await self.push_error(ErrorFrame(f"Error generating TTS: {e}"))
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             await self.stop_ttfb_metrics()
             yield TTSStoppedFrame()
@@ -560,7 +560,8 @@ class SarvamTTSService(InterruptibleTTSService):
             await self._disconnect_websocket()
 
         except Exception as e:
-            logger.error(f"Error during disconnect: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             # Reset state only after everything is cleaned up
             self._started = False
@@ -584,7 +585,8 @@ class SarvamTTSService(InterruptibleTTSService):
 
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"{self} initialization error: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
 
@@ -600,8 +602,8 @@ class SarvamTTSService(InterruptibleTTSService):
             await self._websocket.send(json.dumps(config_message))
             logger.debug("Configuration sent successfully")
         except Exception as e:
-            logger.error(f"Failed to send config: {str(e)}")
-            await self.push_frame(ErrorFrame(f"Failed to send config: {str(e)}"))
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             raise
 
     async def _disconnect_websocket(self):
@@ -614,6 +616,7 @@ class SarvamTTSService(InterruptibleTTSService):
                 await self._websocket.close()
         except Exception as e:
             logger.error(f"{self} error closing websocket: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._started = False
             self._websocket = None
@@ -643,7 +646,7 @@ class SarvamTTSService(InterruptibleTTSService):
                     if "too long" in error_msg.lower() or "timeout" in error_msg.lower():
                         logger.warning("Connection timeout detected, service may need restart")
 
-                    await self.push_frame(ErrorFrame(f"TTS Error: {error_msg}"))
+                    await self.push_frame(ErrorFrame(error=f"TTS Error: {error_msg}"))
 
     async def _keepalive_task_handler(self):
         """Handle keepalive messages to maintain WebSocket connection."""
@@ -699,7 +702,8 @@ class SarvamTTSService(InterruptibleTTSService):
                 await self._send_text(text)
                 await self.start_tts_usage_metrics(text)
             except Exception as e:
-                logger.error(f"{self} error sending message: {e}")
+                logger.error(f"{self} exception: {e}")
+                yield ErrorFrame(error=f"{self} error: {e}")
                 yield TTSStoppedFrame()
                 await self._disconnect()
                 await self._connect()
@@ -707,3 +711,4 @@ class SarvamTTSService(InterruptibleTTSService):
             yield None
         except Exception as e:
             logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")

src/pipecat/services/simli/video.py

@@ -84,6 +84,10 @@ class SimliVideoService(FrameProcessor):
                     Please use 'api_key' and 'face_id' parameters instead.
 
             use_turn_server: Whether to use TURN server for connection. Defaults to False.
+
+                .. deprecated:: 0.0.95
+                    The 'use_turn_server' parameter is deprecated and will be removed in a future version.
+
             latency_interval: Latency interval setting for sending health checks to check
                 the latency to Simli Servers. Defaults to 0.
             simli_url: URL of the simli servers. Can be changed for custom deployments
@@ -135,14 +139,20 @@ class SimliVideoService(FrameProcessor):
 
             config = SimliConfig(**config_kwargs)
 
+        if use_turn_server:
+            warnings.warn(
+                "The 'use_turn_server' parameter is deprecated and will be removed in a future version.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
         self._initialized = False
         # Add buffer time to session limits
         config.maxIdleTime += 5
         config.maxSessionLength += 5
         self._simli_client = SimliClient(
-            config,
-            use_turn_server,
-            latency_interval,
+            config=config,
+            latencyInterval=latency_interval,
             simliURL=simli_url,
         )
 

src/pipecat/services/soniox/stt.py

@@ -327,8 +327,8 @@ class SonioxSTTService(STTService):
             # Expected when closing the connection
             logger.debug("WebSocket connection closed, keepalive task stopped.")
         except Exception as e:
-            logger.error(f"{self} error (_keepalive_task_handler): {e}")
-            await self.push_error(ErrorFrame(f"{self} error (_keepalive_task_handler): {e}"))
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
     async def _receive_task_handler(self):
         if not self._websocket:
@@ -409,7 +409,7 @@ class SonioxSTTService(STTService):
                     )
                     await self.push_error(
                         ErrorFrame(
-                            f"{self} error: {error_code} (_receive_task_handler) - {error_message}"
+                            error=f"{self} error: {error_code} (_receive_task_handler) - {error_message}"
                         )
                     )
 
@@ -425,5 +425,5 @@ class SonioxSTTService(STTService):
             # Expected when closing the connection.
             pass
         except Exception as e:
-            logger.error(f"{self} error: {e}")
-            await self.push_error(ErrorFrame(f"{self} error: {e}"))
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))

src/pipecat/services/speechmatics/stt.py

@@ -467,8 +467,8 @@ class SpeechmaticsSTTService(STTService):
                 await self._client.send_audio(audio)
             yield None
         except Exception as e:
-            logger.error(f"Speechmatics error: {e}")
-            yield ErrorFrame(f"Speechmatics error: {e}", fatal=False)
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
             await self._disconnect()
 
     def update_params(
@@ -514,6 +514,8 @@ class SpeechmaticsSTTService(STTService):
                 self._client.send_message(payload), self.get_event_loop()
             )
         except Exception as e:
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             raise RuntimeError(f"error sending message to STT: {e}")
 
     async def _connect(self) -> None:
@@ -579,7 +581,8 @@ class SpeechmaticsSTTService(STTService):
             logger.debug(f"{self} Connected to Speechmatics STT service")
             await self._call_event_handler("on_connected")
         except Exception as e:
-            logger.error(f"{self} Error connecting to Speechmatics: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
             self._client = None
 
     async def _disconnect(self) -> None:
@@ -593,7 +596,8 @@ class SpeechmaticsSTTService(STTService):
         except asyncio.TimeoutError:
             logger.warning(f"{self} Timeout while closing Speechmatics client connection")
         except Exception as e:
-            logger.error(f"{self} Error closing Speechmatics client: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
         finally:
             self._client = None
             await self._call_event_handler("on_disconnected")

src/pipecat/services/speechmatics/tts.py

@@ -6,6 +6,7 @@
 
 """Speechmatics TTS service integration."""
 
+import asyncio
 from typing import AsyncGenerator, Optional
 from urllib.parse import urlencode
 
@@ -21,6 +22,7 @@ from pipecat.frames.frames import (
     TTSStoppedFrame,
 )
 from pipecat.services.tts_service import TTSService
+from pipecat.utils.network import exponential_backoff_time
 from pipecat.utils.tracing.service_decorators import traced_tts
 
 try:
@@ -43,9 +45,13 @@ class SpeechmaticsTTSService(TTSService):
     SPEECHMATICS_SAMPLE_RATE = 16000
 
     class InputParams(BaseModel):
-        """Optional input parameters for Speechmatics TTS configuration."""
+        """Optional input parameters for Speechmatics TTS configuration.
 
-        pass
+        Parameters:
+            max_retries: Maximum number of retries for TTS requests. Defaults to 5.
+        """
+
+        max_retries: int = 5
 
     def __init__(
         self,
@@ -109,64 +115,119 @@ class SpeechmaticsTTSService(TTSService):
         Yields:
             Frame: Audio frames containing the synthesized speech.
         """
+        # Log the TTS started frame
         logger.debug(f"{self}: Generating TTS [{text}]")
 
+        # HTTP headers
         headers = {
             "Authorization": f"Bearer {self._api_key}",
             "Content-Type": "application/json",
         }
 
+        # HTTP payload
         payload = {
             "text": text,
         }
 
+        # Complete HTTP URL
         url = _get_endpoint_url(self._base_url, self._voice_id, self.sample_rate)
 
         try:
+            # Start TTS TTFB metrics
             await self.start_ttfb_metrics()
 
-            async with self._session.post(url, json=payload, headers=headers) as response:
-                if response.status != 200:
-                    error_message = f"Speechmatics TTS error: HTTP {response.status}"
-                    logger.error(error_message)
-                    yield ErrorFrame(error=error_message)
-                    return
+            # Track attempt
+            attempt = 0
 
-                await self.start_tts_usage_metrics(text)
+            # Keep retrying until we get a 200 response or timeout
+            while True:
+                async with self._session.post(url, json=payload, headers=headers) as response:
+                    """Evaluate response from TTS service."""
 
-                yield TTSStartedFrame()
+                    # 503 : Service unavailable
+                    if response.status == 503:
+                        """Calculate the backoff time and retry."""
 
-                # Process the response in streaming chunks
-                first_chunk = True
-                buffer = b""
+                        try:
+                            # Calculate the backoff time
+                            backoff_time = exponential_backoff_time(
+                                attempt=attempt, min_wait=0.25, max_wait=8.0, multiplier=0.5
+                            )
 
-                async for chunk in response.content.iter_any():
-                    if not chunk:
-                        continue
-                    if first_chunk:
-                        await self.stop_ttfb_metrics()
-                        first_chunk = False
+                            # Increment attempt
+                            attempt += 1
 
-                    buffer += chunk
+                            # Check if we've exceeded the maximum number of attempts
+                            if attempt >= self._params.max_retries:
+                                raise ValueError()
 
-                    # Emit all complete 2-byte int16 samples from buffer
-                    if len(buffer) >= 2:
-                        complete_samples = len(buffer) // 2
-                        complete_bytes = complete_samples * 2
+                            # Report error frame
+                            yield ErrorFrame(
+                                error=f"{self} Service unavailable [503] (attempt {attempt}, retry in {backoff_time:.2f}s)"
+                            )
 
-                        audio_data = buffer[:complete_bytes]
-                        buffer = buffer[complete_bytes:]  # Keep remaining bytes for next iteration
+                            # Wait before retrying
+                            await asyncio.sleep(backoff_time)
 
-                        yield TTSAudioRawFrame(
-                            audio=audio_data,
-                            sample_rate=self.sample_rate,
-                            num_channels=1,
+                            # Retry
+                            continue
+
+                        except (ValueError, ArithmeticError):
+                            yield ErrorFrame(
+                                error=f"{self} Service unavailable [503] (attempts {attempt})",
+                                fatal=True,
+                            )
+                            return
+
+                    # != 200 : Error
+                    if response.status != 200:
+                        yield ErrorFrame(
+                            error=f"{self} Service unavailable [{response.status}]", fatal=True
                         )
+                        return
+
+                    # Update Pipecat metrics
+                    await self.start_tts_usage_metrics(text)
+
+                    # Emit the TTS started frame
+                    yield TTSStartedFrame()
+
+                    # Process the response in streaming chunks
+                    first_chunk = True
+                    buffer = b""
+
+                    # Iterate over each audio data chunk from the TTS API
+                    async for chunk in response.content.iter_any():
+                        if not chunk:
+                            continue
+                        if first_chunk:
+                            await self.stop_ttfb_metrics()
+                            first_chunk = False
+
+                        buffer += chunk
+
+                        # Emit all complete 2-byte int16 samples from buffer
+                        if len(buffer) >= 2:
+                            complete_samples = len(buffer) // 2
+                            complete_bytes = complete_samples * 2
+
+                            audio_data = buffer[:complete_bytes]
+                            buffer = buffer[complete_bytes:]
+
+                            # Emit the audio frame
+                            yield TTSAudioRawFrame(
+                                audio=audio_data,
+                                sample_rate=self.sample_rate,
+                                num_channels=1,
+                            )
+
+                    # Successfully processed the response, break out of retry loop
+                    break
 
         except Exception as e:
-            logger.exception(f"Error generating TTS: {e}")
-            yield ErrorFrame(error=f"Speechmatics TTS error: {str(e)}")
+            yield ErrorFrame(error=f"{self}: Error generating TTS: {e}", fatal=True)
         finally:
+            # Emit the TTS stopped frame
             yield TTSStoppedFrame()
 
 

src/pipecat/services/stt_service.py

@@ -38,7 +38,7 @@ class STTService(AIService):
 
     Event handlers:
         on_connected: Called when connected to the STT service.
-        on_connected: Called when disconnected from the STT service.
+        on_disconnected: Called when disconnected from the STT service.
         on_connection_error: Called when a connection to the STT service error occurs.
 
     Example::

src/pipecat/services/tts_service.py

@@ -12,6 +12,8 @@ from typing import (
     Any,
     AsyncGenerator,
     AsyncIterator,
+    Awaitable,
+    Callable,
     Dict,
     List,
     Mapping,
@@ -23,6 +25,8 @@ from typing import (
 from loguru import logger
 
 from pipecat.frames.frames import (
+    AggregatedTextFrame,
+    AggregationType,
     BotStartedSpeakingFrame,
     BotStoppedSpeakingFrame,
     CancelFrame,
@@ -101,6 +105,16 @@ class TTSService(AIService):
         sample_rate: Optional[int] = None,
         # Text aggregator to aggregate incoming tokens and decide when to push to the TTS.
         text_aggregator: Optional[BaseTextAggregator] = None,
+        # Types of text aggregations that should not be spoken.
+        skip_aggregator_types: Optional[List[str]] = [],
+        # A list of callables to transform text before just before sending it to TTS.
+        # Each callable takes the aggregated text and its type, and returns the transformed text.
+        # To register, provide a list of tuples of (aggregation_type | '*', transform_function).
+        text_transforms: Optional[
+            List[
+                Tuple[AggregationType | str, Callable[[str, str | AggregationType], Awaitable[str]]]
+            ]
+        ] = None,
         # Text filter executed after text has been aggregated.
         text_filters: Optional[Sequence[BaseTextFilter]] = None,
         text_filter: Optional[BaseTextFilter] = None,
@@ -120,6 +134,16 @@ class TTSService(AIService):
             pause_frame_processing: Whether to pause frame processing during audio generation.
             sample_rate: Output sample rate for generated audio.
             text_aggregator: Custom text aggregator for processing incoming text.
+
+                .. deprecated:: 0.0.95
+                    Use an LLMTextProcessor before the TTSService for custom text aggregation.
+
+            skip_aggregator_types: List of aggregation types that should not be spoken.
+            text_transforms: A list of callables to transform text before just before sending it
+                to TTS. Each callable takes the aggregated text and its type, and returns the
+                transformed text. To register, provide a list of tuples of
+                (aggregation_type | '*', transform_function).
+
             text_filters: Sequence of text filters to apply after aggregation.
             text_filter: Single text filter (deprecated, use text_filters).
 
@@ -142,6 +166,21 @@ class TTSService(AIService):
         self._voice_id: str = ""
         self._settings: Dict[str, Any] = {}
         self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
+        if text_aggregator:
+            import warnings
+
+            with warnings.catch_warnings():
+                warnings.simplefilter("always")
+                warnings.warn(
+                    "Parameter 'text_aggregator' is deprecated. Use an LLMTextProcessor before the TTSService for custom text aggregation.",
+                    DeprecationWarning,
+                )
+
+        self._skip_aggregator_types: List[str] = skip_aggregator_types or []
+        self._text_transforms: List[
+            Tuple[AggregationType | str, Callable[[str, AggregationType | str], Awaitable[str]]]
+        ] = text_transforms or []
+        # TODO: Deprecate _text_filters when added to LLMTextProcessor
         self._text_filters: Sequence[BaseTextFilter] = text_filters or []
         self._transport_destination: Optional[str] = transport_destination
         self._tracing_enabled: bool = False
@@ -281,6 +320,39 @@ class TTSService(AIService):
             await self.cancel_task(self._stop_frame_task)
             self._stop_frame_task = None
 
+    def add_text_transformer(
+        self,
+        transform_function: Callable[[str, AggregationType | str], Awaitable[str]],
+        aggregation_type: AggregationType | str = "*",
+    ):
+        """Transform text for a specific aggregation type.
+
+        Args:
+            transform_function: The function to apply for transformation. This function should take
+                the text and aggregation type as input and return the transformed text.
+                Ex.: async def my_transform(text: str, aggregation_type: str) -> str:
+            aggregation_type: The type of aggregation to transform. This value defaults to "*" indicating
+                the function should handle all text before sending to TTS.
+        """
+        self._text_transforms.append((aggregation_type, transform_function))
+
+    def remove_text_transformer(
+        self,
+        transform_function: Callable[[str, AggregationType | str], Awaitable[str]],
+        aggregation_type: AggregationType | str = "*",
+    ):
+        """Remove a text transformer for a specific aggregation type.
+
+        Args:
+            transform_function: The function to remove.
+            aggregation_type: The type of aggregation to remove the transformer for.
+        """
+        self._text_transforms = [
+            (agg_type, func)
+            for agg_type, func in self._text_transforms
+            if not (agg_type == aggregation_type and func == transform_function)
+        ]
+
     async def _update_settings(self, settings: Mapping[str, Any]):
         for key, value in settings.items():
             if key in self._settings:
@@ -336,6 +408,8 @@ class TTSService(AIService):
             and frame.skip_tts
         ):
             await self.push_frame(frame, direction)
+        elif isinstance(frame, AggregatedTextFrame):
+            await self._push_tts_frames(frame)
         elif (
             isinstance(frame, TextFrame)
             and not isinstance(frame, InterimTranscriptionFrame)
@@ -351,10 +425,16 @@ class TTSService(AIService):
             # pause to avoid audio overlapping.
             await self._maybe_pause_frame_processing()
 
-            sentence = self._text_aggregator.text
+            pending_aggregation = self._text_aggregator.text
+
+            # Reset aggregator state
             await self._text_aggregator.reset()
             self._processing_text = False
-            await self._push_tts_frames(sentence)
+
+            if pending_aggregation.text:
+                await self._push_tts_frames(
+                    AggregatedTextFrame(pending_aggregation.text, pending_aggregation.type)
+                )
             if isinstance(frame, LLMFullResponseEndFrame):
                 if self._push_text_frames:
                     await self.push_frame(frame, direction)
@@ -363,7 +443,8 @@ class TTSService(AIService):
         elif isinstance(frame, TTSSpeakFrame):
             # Store if we were processing text or not so we can set it back.
             processing_text = self._processing_text
-            await self._push_tts_frames(frame.text)
+            # Assumption: text in TTSSpeakFrame does not include inter-frame spaces
+            await self._push_tts_frames(AggregatedTextFrame(frame.text, AggregationType.SENTENCE))
             # We pause processing incoming frames because we are sending data to
             # the TTS. We pause to avoid audio overlapping.
             await self._maybe_pause_frame_processing()
@@ -453,15 +534,35 @@ class TTSService(AIService):
 
     async def _process_text_frame(self, frame: TextFrame):
         text: Optional[str] = None
+        includes_inter_frame_spaces: bool = False
         if not self._aggregate_sentences:
             text = frame.text
+            includes_inter_frame_spaces = frame.includes_inter_frame_spaces
+            aggregated_by = "token"
         else:
-            text = await self._text_aggregator.aggregate(frame.text)
+            aggregate = await self._text_aggregator.aggregate(frame.text)
+            if aggregate:
+                text = aggregate.text
+                aggregated_by = aggregate.type
 
         if text:
-            await self._push_tts_frames(text)
+            logger.trace(f"Pushing TTS frames for text: {text}, {aggregated_by}")
+            await self._push_tts_frames(
+                AggregatedTextFrame(text, aggregated_by), includes_inter_frame_spaces
+            )
+
+    async def _push_tts_frames(
+        self, src_frame: AggregatedTextFrame, includes_inter_frame_spaces: Optional[bool] = False
+    ):
+        type = src_frame.aggregated_by
+        text = src_frame.text
+
+        # Skip sending to TTS if the aggregation type is in the skip list. Simply
+        # push the original frame downstream.
+        if type in self._skip_aggregator_types:
+            await self.push_frame(src_frame)
+            return
 
-    async def _push_tts_frames(self, text: str):
         # Remove leading newlines only
         text = text.lstrip("\n")
 
@@ -477,20 +578,46 @@ class TTSService(AIService):
 
         await self.start_processing_metrics()
 
-        # Process all filter.
+        # Process all filters.
         for filter in self._text_filters:
             await filter.reset_interruption()
             text = await filter.filter(text)
 
-        if text:
-            await self.process_generator(self.run_tts(text))
+        if not text.strip():
+            await self.stop_processing_metrics()
+            return
+
+        # To support use cases that may want to know the text before it's spoken, we
+        # push the AggregatedTextFrame version before transforming and sending to TTS.
+        # However, we do not want to add this text to the assistant context until it
+        # is spoken, so we set append_to_context to False.
+        src_frame.append_to_context = False
+        await self.push_frame(src_frame)
+
+        # Note: Text transformations are meant to only affect the text sent to the TTS for
+        # TTS-specific purposes. This allows for explicit TTS modifications (e.g., inserting
+        # TTS supported tags for spelling or emotion or replacing an @ with "at"). For TTS
+        # services that support word-level timestamps, this CAN affect the resulting context
+        # since the TTSTextFrames are generated from the TTS output stream
+        transformed_text = text
+        for aggregation_type, transform in self._text_transforms:
+            if aggregation_type == type or aggregation_type == "*":
+                transformed_text = await transform(transformed_text, type)
+        await self.process_generator(self.run_tts(transformed_text))
 
         await self.stop_processing_metrics()
 
         if self._push_text_frames:
-            # We send the original text after the audio. This way, if we are
-            # interrupted, the text is not added to the assistant context.
-            await self.push_frame(TTSTextFrame(text))
+            # In TTS services that support word timestamps, the TTSTextFrames
+            # are pushed as words are spoken. However, in the case where the TTS service
+            # does not support word timestamps (i.e. _push_text_frames is True), we send
+            # the original (non-transformed) text after the TTS generation has completed.
+            # This way, if we are interrupted, the text is not added to the assistant
+            # context and the context that IS added does not include TTS-specific tags
+            # or transformations.
+            frame = TTSTextFrame(text, aggregated_by=type)
+            frame.includes_inter_frame_spaces = includes_inter_frame_spaces
+            await self.push_frame(frame)
 
     async def _stop_frame_handler(self):
         has_started = False
@@ -616,7 +743,9 @@ class WordTTSService(TTSService):
                 frame = TTSStoppedFrame()
                 frame.pts = last_pts
             else:
-                frame = TTSTextFrame(word)
+                # Assumption: word-by-word text frames don't include spaces, so
+                # we can rely on the default includes_inter_frame_spaces=False
+                frame = TTSTextFrame(word, aggregated_by=AggregationType.WORD)
                 frame.pts = self._initial_word_timestamp + timestamp
             if frame:
                 last_pts = frame.pts

src/pipecat/services/ultravox/stt.py

@@ -246,7 +246,8 @@ class UltravoxSTTService(AIService):
 
             logger.info("Model warm-up completed successfully")
         except Exception as e:
-            logger.warning(f"Model warm-up failed: {e}")
+            logger.error(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
 
     def _generate_silent_audio(self, sample_rate=16000, duration_sec=1.0):
         """Generate silent audio as a numpy array.
@@ -376,7 +377,7 @@ class UltravoxSTTService(AIService):
                             if arr.size > 0:  # Check if array is not empty
                                 audio_arrays.append(arr)
                         except Exception as e:
-                            logger.error(f"Error processing bytes audio frame: {e}")
+                            yield ErrorFrame(error=f"{self} error: {e}")
                     # Handle numpy array data
                     elif isinstance(f.audio, np.ndarray):
                         if f.audio.size > 0:  # Check if array is not empty
@@ -436,14 +437,14 @@ class UltravoxSTTService(AIService):
                     yield LLMFullResponseEndFrame()
 
                 except Exception as e:
-                    logger.error(f"Error generating text from model: {e}")
-                    yield ErrorFrame(f"Error generating text: {str(e)}")
+                    logger.error(f"{self} exception: {e}")
+                    yield ErrorFrame(error=f"{self} error: {e}")
             else:
-                logger.warning("No model available for text generation")
+                logger.error("No model available for text generation")
                 yield ErrorFrame("No model available for text generation")
 
         except Exception as e:
-            logger.error(f"Error processing audio buffer: {e}")
+            logger.error(f"{self} exception: {e}")
             import traceback
 
             logger.error(traceback.format_exc())

src/pipecat/services/websocket_service.py

@@ -36,6 +36,7 @@ class WebsocketService(ABC):
         """
         self._websocket: Optional[websockets.WebSocketClientProtocol] = None
         self._reconnect_on_error = reconnect_on_error
+        self._reconnect_in_progress: bool = False  # Add this flag
 
     async def _verify_connection(self) -> bool:
         """Verify the websocket connection is active and responsive.
@@ -66,6 +67,59 @@ class WebsocketService(ABC):
         await self._connect_websocket()
         return await self._verify_connection()
 
+    async def _try_reconnect(
+        self,
+        max_retries: int = 3,
+        report_error: Optional[Callable[[ErrorFrame], Awaitable[None]]] = None,
+    ) -> bool:
+        # Prevent concurrent reconnection attempts
+        if self._reconnect_in_progress:
+            logger.warning(f"{self} reconnect attempt aborted: already in progress")
+            return False
+
+        self._reconnect_in_progress = True
+        last_exception: Optional[Exception] = None
+        try:
+            for attempt in range(1, max_retries + 1):
+                try:
+                    logger.warning(f"{self} reconnecting, attempt {attempt}")
+                    if await self._reconnect_websocket(attempt):
+                        logger.info(f"{self} reconnected successfully on attempt {attempt}")
+                        return True
+                except Exception as e:
+                    last_exception = e
+                    logger.error(f"{self} reconnection attempt {attempt} failed: {e}")
+                    if report_error:
+                        await report_error(
+                            ErrorFrame(f"{self} reconnection attempt {attempt} failed: {e}")
+                        )
+                wait_time = exponential_backoff_time(attempt)
+                await asyncio.sleep(wait_time)
+            fatal_msg = f"{self} failed to reconnect after {max_retries} attempts"
+            if last_exception:
+                fatal_msg += f": {last_exception}"
+            logger.error(fatal_msg)
+            if report_error:
+                await report_error(ErrorFrame(fatal_msg, fatal=True))
+            return False
+        finally:
+            self._reconnect_in_progress = False
+
+    async def send_with_retry(self, message, report_error: Callable[[ErrorFrame], Awaitable[None]]):
+        """Attempt to send a message, retrying after reconnect if necessary."""
+        try:
+            await self._websocket.send(message)
+        except Exception as e:
+            logger.error(f"{self} send failed: {e}, will try to reconnect")
+            # Try to reconnect before retrying
+            success = await self._try_reconnect(report_error=report_error)
+            if success:
+                logger.info(f"{self} reconnected successfully, will retry send the message")
+                # trying to send the message one more time
+                await self._websocket.send(message)
+            else:
+                logger.error(f"{self} send failed; unable to reconnect")
+
     async def _receive_task_handler(self, report_error: Callable[[ErrorFrame], Awaitable[None]]):
         """Handle websocket message receiving with automatic retry logic.
 
@@ -76,13 +130,9 @@ class WebsocketService(ABC):
         Args:
             report_error: Callback function to report connection errors.
         """
-        retry_count = 0
-        MAX_RETRIES = 3
-
         while True:
             try:
                 await self._receive_messages()
-                retry_count = 0  # Reset counter on successful message receive
             except ConnectionClosedOK as e:
                 # Normal closure, don't retry
                 logger.debug(f"{self} connection closed normally: {e}")
@@ -92,21 +142,9 @@ class WebsocketService(ABC):
                 logger.error(message)
 
                 if self._reconnect_on_error:
-                    retry_count += 1
-                    if retry_count >= MAX_RETRIES:
-                        await report_error(ErrorFrame(message, fatal=True))
+                    success = await self._try_reconnect(report_error=report_error)
+                    if not success:
                         break
-
-                    logger.warning(f"{self} connection error, will retry: {e}")
-                    await report_error(ErrorFrame(message))
-
-                    try:
-                        if await self._reconnect_websocket(retry_count):
-                            retry_count = 0  # Reset counter on successful reconnection
-                        wait_time = exponential_backoff_time(retry_count)
-                        await asyncio.sleep(wait_time)
-                    except Exception as reconnect_error:
-                        logger.error(f"{self} reconnection failed: {reconnect_error}")
                 else:
                     await report_error(ErrorFrame(message))
                     break

src/pipecat/services/whisper/base_stt.py

@@ -226,8 +226,8 @@ class BaseWhisperSTTService(SegmentedSTTService):
                 logger.warning("Received empty transcription from API")
 
         except Exception as e:
-            logger.exception(f"Exception during transcription: {e}")
-            yield ErrorFrame(f"Error during transcription: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
 
     async def _transcribe(self, audio: bytes) -> Transcription:
         """Transcribe audio data to text.

src/pipecat/services/whisper/stt.py

@@ -428,5 +428,5 @@ class WhisperSTTServiceMLX(WhisperSTTService):
                 )
 
         except Exception as e:
-            logger.exception(f"MLX Whisper transcription error: {e}")
-            yield ErrorFrame(f"MLX Whisper transcription error: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")

src/pipecat/services/xtts/tts.py

@@ -146,7 +146,7 @@ class XTTSService(TTSService):
                 )
                 await self.push_error(
                     ErrorFrame(
-                        f"Error error getting studio speakers (status: {r.status}, error: {text})"
+                        error=f"Error getting studio speakers (status: {r.status}, error: {text})"
                     )
                 )
                 return
@@ -187,7 +187,7 @@ class XTTSService(TTSService):
             if r.status != 200:
                 text = await r.text()
                 logger.error(f"{self} error getting audio (status: {r.status}, error: {text})")
-                yield ErrorFrame(f"Error getting audio (status: {r.status}, error: {text})")
+                yield ErrorFrame(error=f"Error getting audio (status: {r.status}, error: {text})")
                 return
 
             await self.start_tts_usage_metrics(text)

src/pipecat/tests/utils.py

@@ -203,8 +203,16 @@ async def run_test(
             if not isinstance(frame, EndFrame) or not send_end_frame:
                 received_down_frames.append(frame)
 
-        print("received DOWN frames =", received_down_frames)
-        print("expected DOWN frames =", expected_down_frames)
+        down_frames_printed = "["
+        for frame in received_down_frames:
+            down_frames_printed += f"{frame.__class__.__name__}, "
+        down_frames_printed += "]"
+        expected_frames_printed = "["
+        for frame in expected_down_frames:
+            expected_frames_printed += f"{frame.__name__}, "
+        expected_frames_printed += "]"
+        print("received DOWN frames =", down_frames_printed)
+        print("expected DOWN frames =", expected_frames_printed)
 
         assert len(received_down_frames) == len(expected_down_frames)
 

src/pipecat/transcriptions/language.py

@@ -64,7 +64,9 @@ class Language(StrEnum):
     AR_SA = "ar-SA"
     AR_SY = "ar-SY"
     AR_TN = "ar-TN"
+    AR_XA = "ar-XA"
     AR_YE = "ar-YE"
+    AR_001 = "ar-001"
 
     # Assamese
     AS = "as"
@@ -82,6 +84,7 @@ class Language(StrEnum):
 
     # Belarusian
     BE = "be"
+    BE_BY = "be-BY"
 
     # Bulgarian
     BG = "bg"
@@ -108,6 +111,7 @@ class Language(StrEnum):
 
     # Cebuano
     CEB = "ceb"
+    CEB_PH = "ceb-PH"
 
     # Mandarin Chinese
     CMN = "cmn"
@@ -180,6 +184,7 @@ class Language(StrEnum):
     ES_US = "es-US"
     ES_UY = "es-UY"
     ES_VE = "es-VE"
+    ES_419 = "es-419"
 
     # Estonian
     ET = "et"
@@ -249,6 +254,7 @@ class Language(StrEnum):
 
     # Haitian Creole
     HT = "ht"
+    HT_HT = "ht-HT"
 
     # Hungarian
     HU = "hu"
@@ -287,6 +293,7 @@ class Language(StrEnum):
     # Javanese
     JV = "jv"
     JV_ID = "jv-ID"
+    JV_JV = "jv-JV"
     JW = "jw"  # Fal requires for Javanese
 
     # Georgian
@@ -308,6 +315,10 @@ class Language(StrEnum):
     KN = "kn"
     KN_IN = "kn-IN"
 
+    # Konkani
+    KOK = "kok"
+    KOK_IN = "kok-IN"
+
     # Korean
     KO = "ko"
     KO_KR = "ko-KR"
@@ -321,9 +332,11 @@ class Language(StrEnum):
 
     # Latin
     LA = "la"
+    LA_VA = "la-VA"
 
     # Luxembourgish
     LB = "lb"
+    LB_LU = "lb-LU"
 
     # Lingala
     LN = "ln"
@@ -348,6 +361,7 @@ class Language(StrEnum):
 
     # Malagasy
     MG = "mg"
+    MG_MG = "mg-MG"
 
     # Maori
     MI = "mi"
@@ -356,6 +370,10 @@ class Language(StrEnum):
     MK = "mk"
     MK_MK = "mk-MK"
 
+    # Maithili
+    MAI = "mai"
+    MAI_IN = "mai-IN"
+
     # Malayalam
     ML = "ml"
     ML_IN = "ml-IN"
@@ -386,6 +404,7 @@ class Language(StrEnum):
     NB_NO = "nb-NO"
     NO = "no"
     NN = "nn"  # Norwegian Nynorsk
+    NN_NO = "nn-NO"
 
     # Nepali
     NE = "ne"
@@ -439,6 +458,7 @@ class Language(StrEnum):
 
     # Sindhi
     SD = "sd"
+    SD_IN = "sd-IN"
 
     # Sinhala
     SI = "si"

src/pipecat/transports/livekit/utils.py

@@ -0,0 +1,96 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""LiveKit REST Helpers.
+
+Methods that wrap the LiveKit API for room management.
+"""
+
+import aiohttp
+
+
+class LiveKitRESTHelper:
+    """Helper class for interacting with LiveKit's REST API.
+
+    Provides methods for managing LiveKit rooms.
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        api_secret: str,
+        api_url: str = "https://your-livekit-host.com",
+        aiohttp_session: aiohttp.ClientSession,
+    ):
+        """Initialize the LiveKit REST helper.
+
+        Args:
+            api_key: Your LiveKit API key.
+            api_secret: Your LiveKit API secret.
+            api_url: LiveKit server URL (e.g. "https://your-livekit-host.com").
+            aiohttp_session: Async HTTP session for making requests.
+        """
+        self.api_key = api_key
+        self.api_secret = api_secret
+        self.api_url = api_url.rstrip("/")
+        self.aiohttp_session = aiohttp_session
+
+    def _create_access_token(self, room_create: bool = True) -> str:
+        """Create a signed access token for LiveKit API authentication.
+
+        Args:
+            room_create: Whether to grant roomCreate permission.
+
+        Returns:
+            Signed JWT access token.
+        """
+        import time
+
+        import jwt
+
+        claims = {
+            "iss": self.api_key,
+            "sub": self.api_key,
+            "nbf": int(time.time()),
+            "exp": int(time.time()) + 60,  # Token valid for 60 seconds
+            "video": {
+                "roomCreate": room_create,
+            },
+        }
+
+        return jwt.encode(claims, self.api_secret, algorithm="HS256")
+
+    async def delete_room_by_name(self, room_name: str) -> bool:
+        """Delete a LiveKit room by name.
+
+        This will forcibly disconnect all participants currently in the room.
+
+        Args:
+            room_name: Name of the room to delete.
+
+        Returns:
+            True if deletion was successful.
+
+        Raises:
+            Exception: If deletion fails.
+        """
+        token = self._create_access_token(room_create=True)
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+        }
+
+        async with self.aiohttp_session.post(
+            f"{self.api_url}/twirp/livekit.RoomService/DeleteRoom",
+            headers=headers,
+            json={"room": room_name},
+        ) as r:
+            if r.status != 200:
+                text = await r.text()
+                raise Exception(f"Failed to delete room [{room_name}] (status: {r.status}): {text}")
+
+        return True

src/pipecat/utils/string.py

@@ -18,6 +18,7 @@ Dependencies:
 """
 
 import re
+from dataclasses import dataclass
 from typing import FrozenSet, List, Optional, Sequence, Tuple
 
 import nltk
@@ -198,7 +199,24 @@ def parse_start_end_tags(
     return (None, current_tag_index)
 
 
-def concatenate_aggregated_text(text_parts: List[str]) -> str:
+@dataclass
+class TextPartForConcatenation:
+    """Class representing a part of text for concatenation with concatenate_aggregated_text.
+
+    Attributes:
+        text: The text content.
+        includes_inter_part_spaces: Whether any necessary inter-frame
+            (leading/trailing) spaces are already included in the text.
+    """
+
+    text: str
+    includes_inter_part_spaces: bool
+
+    def __str__(self):
+        return f"{self.name}(text: [{self.text}], includes_inter_part_spaces: {self.includes_inter_part_spaces})"
+
+
+def concatenate_aggregated_text(text_parts: List[TextPartForConcatenation]) -> str:
     """Concatenate a list of text parts into a single string.
 
     This function joins the provided list of text parts into a single string,
@@ -208,54 +226,55 @@ def concatenate_aggregated_text(text_parts: List[str]) -> str:
     transcription services.
 
     Args:
-        text_parts: A list of strings representing parts of text to concatenate.
+        text_parts: A list of text parts to concatenate.
 
     Returns:
         A single concatenated string.
     """
-    # Check specifically for space characters, previously isspace() was used
-    # but that includes all whitespace characters (e.g. \n), not just spaces.
-    has_leading_spaces = any(part and part[0] == " " for part in text_parts[1:])
-    has_trailing_spaces = any(part and part[-1] == " " for part in text_parts[:-1])
+    result = ""
+    last_includes_inter_part_spaces = False
 
-    # Check for trailing non-space whitespace (e.g., \n, \r, \t) which indicates
-    # syllable-by-syllable output with line breaks.
-    # Example: Gemini Live: ["Met", "amo", "rph", "osi", "s.\n"]
-    has_trailing_whitespace = any(
-        part and part[-1] != " " and part[-1].isspace() for part in text_parts
-    )
+    if not text_parts:
+        return result
 
-    # Check if we have punctuation-only fragments, which indicates syllable-by-syllable
-    # output where punctuation arrives as a separate fragment.
-    # Example: OpenAI Realtime single word: ["Met", "am", "orph", "osis", "."]
-    punctuation_chars = ".,!?;:—-'\"…"
-    has_punctuation_only = any(
-        part and len(part.strip()) == 0 or all(c in punctuation_chars for c in part)
-        for part in text_parts
-    )
+    def append_part(part: TextPartForConcatenation):
+        nonlocal result
+        nonlocal last_includes_inter_part_spaces
+        result += part.text
+        last_includes_inter_part_spaces = part.includes_inter_part_spaces
 
-    # If there are embedded spaces or other whitespace in the fragments, use direct concatenation
-    contains_spacing_between_fragments = (
-        has_leading_spaces or has_trailing_spaces or has_trailing_whitespace
-    )
+    for part in text_parts:
+        # Part is empty.
+        # Skip.
+        if not part.text:
+            continue
 
-    # Apply corresponding joining method based on detected spacing patterns:
+        # Result is as yet empty.
+        # Just append.
+        if not result:
+            append_part(part)
+            continue
 
-    if has_punctuation_only and not contains_spacing_between_fragments:
-        # Syllable-by-syllable output with standalone punctuation fragment. Examples:
-        # - OpenAI Realtime: ["Met", "am", "orph", "osis", "."] → "Metamorphosis."
-        result = "".join(text_parts)
-    elif contains_spacing_between_fragments:
-        # Fragments already have embedded spacing or trailing whitespace - concatenate directly. Examples:
-        # - OpenAI Realtime: ['Hey', ' there', '!', ' Great', ' to', ' meet', ' you', '!']
-        # - Gemini Live (spaces): ['Hel', 'lo.', ' Wo', 'u', 'ld ', 'you', ' li', 'ke ', 'to ', 'he', 'ar a joke?\n']
-        # - Gemini Live (newline): ["Met", "amo", "rph", "osi", "s.\n"] → "Metamorphosis."
-        # - Sentence level TTS services: ['Hello!', ' How can I assist you today?']
-        result = "".join(text_parts)
-    else:
-        # Word-by-word fragments without spacing - join with spaces. Examples:
-        # - Word level TTS services: ["Hello", "there.", "How", "are", "you?"] → "Hello there. How are you?"
-        result = " ".join(text_parts)
+        if part.includes_inter_part_spaces and last_includes_inter_part_spaces:
+            # This part is part of an ongoing run that has spaces already included.
+            # Just append.
+            append_part(part)
+        elif not part.includes_inter_part_spaces and not last_includes_inter_part_spaces:
+            # This part is part of an ongoing run that has no spaces included.
+            # Add a space before appending.
+            result += " "
+            append_part(part)
+        else:
+            # This part represents a transition to a new run (spaces -> no spaces, or vice versa).
+            # Add a space if needed, before appending.
+            if not result[-1].isspace() and not part.text[0].isspace():
+                result += " "
+            append_part(part)
+
+    # NOTE: the above logic assumes that runs of text parts with
+    # includes_inter_part_spaces=True are well-formed, i.e. they're not
+    # actually multiple separate runs with a space-less boundary, like
+    # "hello ", "world.", "goodnight ", "moon."
 
     # Clean up any excessive whitespace
     result = result.strip()

src/pipecat/utils/text/base_text_aggregator.py

@@ -12,9 +12,47 @@ aggregated text should be sent for speech synthesis.
 """
 
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
 from typing import Optional
 
 
+class AggregationType(str, Enum):
+    """Built-in aggregation strings."""
+
+    SENTENCE = "sentence"
+    WORD = "word"
+
+    def __str__(self):
+        return self.value
+
+
+@dataclass
+class Aggregation:
+    """Data class representing aggregated text and its type.
+
+    An Aggregation object is created whenever a stream of text is aggregated by
+    a text aggregator. It contains the aggregated text and a type indicating
+    the nature of the aggregation.
+
+    Parameters:
+        text: The aggregated text content.
+        type: The type of aggregation the text represents (e.g., 'sentence', 'word', 'token',
+              'my_custom_aggregation').
+    """
+
+    text: str
+    type: str
+
+    def __str__(self) -> str:
+        """Return a string representation of the aggregation.
+
+        Returns:
+            A descriptive string showing the type and text of the aggregation.
+        """
+        return f"Aggregation by {self.type}: {self.text}"
+
+
 class BaseTextAggregator(ABC):
     """Base class for text aggregators in the Pipecat framework.
 
@@ -30,7 +68,7 @@ class BaseTextAggregator(ABC):
 
     @property
     @abstractmethod
-    def text(self) -> str:
+    def text(self) -> Aggregation:
         """Get the currently aggregated text.
 
         Subclasses must implement this property to return the text that has
@@ -42,25 +80,33 @@ class BaseTextAggregator(ABC):
         pass
 
     @abstractmethod
-    async def aggregate(self, text: str) -> Optional[str]:
+    async def aggregate(self, text: str) -> Optional[Aggregation]:
         """Aggregate the specified text with the currently accumulated text.
 
         This method should be implemented to define how the new text contributes
-        to the aggregation process. It returns the updated aggregated text if
-        it's ready to be processed, or None otherwise.
+        to the aggregation process. It returns the aggregated text and a string
+        describing how it was aggregated if it's ready to be processed,
+        or None otherwise.
 
         Subclasses should implement their specific logic for:
 
         - How to combine new text with existing accumulated text
         - When to consider the aggregated text ready for processing
         - What criteria determine text completion (e.g., sentence boundaries)
+        - When a completion occurs, the method should return an Aggregation object
+          containing the aggregated text and its type. The text should be stripped
+          of leading/trailing whitespace so that consumers can rely on a consistent
+          format.
 
         Args:
             text: The text to be aggregated.
 
         Returns:
-            The updated aggregated text if ready for processing, or None if more
-            text is needed before the aggregated content is ready.
+            An Aggregation object if ready for processing, or None if more
+            text is needed before the aggregated content is ready. If an Aggregation
+            object is returned, it should consist of the updated aggregated text,
+            stripped of leading/trailing whitespace, and a string indicating the
+            type of aggregation (e.g., 'sentence', 'word', 'token', 'my_custom_aggregation').
         """
         pass
 

src/pipecat/utils/text/base_text_filter.py

@@ -26,7 +26,6 @@ class BaseTextFilter(ABC):
     behavior, settings management, and interruption handling logic.
     """
 
-    @abstractmethod
     async def update_settings(self, settings: Mapping[str, Any]):
         """Update the filter's configuration settings.
 
@@ -53,7 +52,6 @@ class BaseTextFilter(ABC):
         """
         pass
 
-    @abstractmethod
     async def handle_interruption(self):
         """Handle interruption events in the processing pipeline.
 
@@ -62,7 +60,6 @@ class BaseTextFilter(ABC):
         """
         pass
 
-    @abstractmethod
     async def reset_interruption(self):
         """Reset the filter state after an interruption has been handled.
 

src/pipecat/utils/text/pattern_pair_aggregator.py

@@ -8,19 +8,41 @@
 
 This module provides an aggregator that identifies and processes content between
 pattern pairs (like XML tags or custom delimiters) in streaming text, with
-support for custom handlers and configurable pattern removal.
+support for custom handlers and configurable actions for when a pattern is found.
 """
 
 import re
-from typing import Awaitable, Callable, Optional, Tuple
+from enum import Enum
+from typing import Awaitable, Callable, List, Optional, Tuple
 
 from loguru import logger
 
 from pipecat.utils.string import match_endofsentence
-from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
+from pipecat.utils.text.base_text_aggregator import Aggregation, AggregationType, BaseTextAggregator
 
 
-class PatternMatch:
+class MatchAction(Enum):
+    """Actions to take when a pattern pair is matched.
+
+    Parameters:
+        REMOVE: The text along with its delimiters will be removed from the streaming text.
+              Sentence aggregation will continue on as if this text did not exist.
+        KEEP: The delimiters will be removed, but the content between them will be kept.
+              Sentence aggregation will continue on with the internal text included.
+        AGGREGATE: The delimiters will be removed and the content between will be treated
+              as a separate aggregation. Any text before the start of the pattern will be
+              returned early, whether or not a complete sentence was found. Then the pattern
+              will be returned. Then the aggregation will continue on sentence matching after
+              the closing delimiter is found. The content between the delimiters is not
+              aggregated by sentence. It is aggregated as one single block of text.
+    """
+
+    REMOVE = "remove"
+    KEEP = "keep"
+    AGGREGATE = "aggregate"
+
+
+class PatternMatch(Aggregation):
     """Represents a matched pattern pair with its content.
 
     A PatternMatch object is created when a complete pattern pair is found
@@ -29,25 +51,25 @@ class PatternMatch:
     content between the patterns.
     """
 
-    def __init__(self, pattern_id: str, full_match: str, content: str):
+    def __init__(self, content: str, type: str, full_match: str):
         """Initialize a pattern match.
 
         Args:
-            pattern_id: The identifier of the matched pattern pair.
+            type: The type of the matched pattern pair. It should be representative
+                   of the content type (e.g., 'sentence', 'code', 'speaker', 'custom').
             full_match: The complete text including start and end patterns.
             content: The text content between the start and end patterns.
         """
-        self.pattern_id = pattern_id
+        super().__init__(text=content, type=type)
         self.full_match = full_match
-        self.content = content
 
     def __str__(self) -> str:
         """Return a string representation of the pattern match.
 
         Returns:
-            A descriptive string showing the pattern ID and content.
+            A descriptive string showing the pattern type and content.
         """
-        return f"PatternMatch(id={self.pattern_id}, content={self.content})"
+        return f"PatternMatch(type={self.type}, text={self.text}, full_match={self.full_match})"
 
 
 class PatternPairAggregator(BaseTextAggregator):
@@ -55,16 +77,21 @@ class PatternPairAggregator(BaseTextAggregator):
 
     This aggregator buffers text until it can identify complete pattern pairs
     (defined by start and end patterns), processes the content between these
-    patterns using registered handlers, and returns text at sentence boundaries.
-    It's particularly useful for processing structured content in streaming text,
-    such as XML tags, markdown formatting, or custom delimiters.
+    patterns using registered handlers. By default, its aggregation method
+    returns text at sentence boundaries, and remove the content found between
+    any matched patterns. However, matched patterns can also be configured to
+    returned as a separate aggregation object containing the content between
+    their start and end patterns or left in, so that only the delimiters are
+    removed and a callback can be triggered.
+
+    This aggregator is particularly useful for processing structured content in
+    streaming text, such as XML tags, markdown formatting, or custom delimiters.
 
     The aggregator ensures that patterns spanning multiple text chunks are
-    correctly identified and handles cases where patterns contain sentence
-    boundaries.
+    correctly identified.
     """
 
-    def __init__(self):
+    def __init__(self, **kwargs):
         """Initialize the pattern pair aggregator.
 
         Creates an empty aggregator with no patterns or handlers registered.
@@ -75,16 +102,27 @@ class PatternPairAggregator(BaseTextAggregator):
         self._handlers = {}
 
     @property
-    def text(self) -> str:
-        """Get the currently buffered text.
+    def text(self) -> Aggregation:
+        """Get the currently aggregated text.
 
         Returns:
-            The current text buffer content that hasn't been processed yet.
+            The text that has been accumulated in the buffer.
         """
-        return self._text
+        pattern_start = self._match_start_of_pattern(self._text)
+        stripped_text = self._text.strip()
+        type = (
+            pattern_start[1].get("type", AggregationType.SENTENCE)
+            if pattern_start
+            else AggregationType.SENTENCE
+        )
+        return Aggregation(text=stripped_text, type=type)
 
-    def add_pattern_pair(
-        self, pattern_id: str, start_pattern: str, end_pattern: str, remove_match: bool = True
+    def add_pattern(
+        self,
+        type: str,
+        start_pattern: str,
+        end_pattern: str,
+        action: MatchAction = MatchAction.REMOVE,
     ) -> "PatternPairAggregator":
         """Add a pattern pair to detect in the text.
 
@@ -93,41 +131,94 @@ class PatternPairAggregator(BaseTextAggregator):
         the end pattern, and treat the content between them as a match.
 
         Args:
-            pattern_id: Unique identifier for this pattern pair.
+            type: Identifier for this pattern pair. Should be unique and ideally descriptive.
+                  (e.g., 'code', 'speaker', 'custom'). type can not be 'sentence' or 'word' as
+                  those are reserved for the default behavior.
             start_pattern: Pattern that marks the beginning of content.
             end_pattern: Pattern that marks the end of content.
-            remove_match: Whether to remove the matched content from the text.
+            action: What to do when a complete pattern is matched:
+                    - MatchAction.REMOVE: Remove the matched pattern from the text.
+                    - MatchAction.KEEP: Keep the matched pattern in the text and treat it as
+                                        normal text. This allows you to register handlers for
+                                        the pattern without affecting the aggregation logic.
+                    - MatchAction.AGGREGATE: Return the matched pattern as a separate
+                                             aggregation object.
 
         Returns:
             Self for method chaining.
         """
-        self._patterns[pattern_id] = {
+        if type in [AggregationType.SENTENCE, AggregationType.WORD]:
+            raise ValueError(
+                f"The aggregation type '{type}' is reserved for default behavior and can not be used for custom patterns."
+            )
+        self._patterns[type] = {
             "start": start_pattern,
             "end": end_pattern,
-            "remove_match": remove_match,
+            "type": type,
+            "action": action,
         }
         return self
 
+    def add_pattern_pair(
+        self, pattern_id: str, start_pattern: str, end_pattern: str, remove_match: bool = True
+    ):
+        """Add a pattern pair to detect in the text.
+
+        .. deprecated:: 0.0.95
+            This function is deprecated and will be removed in a future version.
+            Use `add_pattern` with a type and MatchAction instead.
+
+            This method calls `add_pattern` setting type with the provided pattern_id and action
+            to either MatchAction.REMOVE or MatchAction.KEEP based on `remove_match`.
+
+        Args:
+            pattern_id: Identifier for this pattern pair. Should be unique and ideally descriptive.
+                        (e.g., 'code', 'speaker', 'custom'). pattern_id can not be 'sentence' or 'word'
+                        as those arereserved for the default behavior.
+            start_pattern: Pattern that marks the beginning of content.
+            end_pattern: Pattern that marks the end of content.
+            remove_match: If True, the matched pattern will be removed from the text. (Same as MatchAction.REMOVE)
+                          If False, it will be kept and treated as normal text. (Same as MatchAction.KEEP)
+        """
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("once")
+            warnings.warn(
+                "add_pattern_pair with a pattern_id or remove_match is deprecated and will be"
+                " removed in a future version. Use add_pattern with a type and MatchAction instead",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
+        action = MatchAction.REMOVE if remove_match else MatchAction.KEEP
+        return self.add_pattern(
+            type=pattern_id,
+            start_pattern=start_pattern,
+            end_pattern=end_pattern,
+            action=action,
+        )
+
     def on_pattern_match(
-        self, pattern_id: str, handler: Callable[[PatternMatch], Awaitable[None]]
+        self, type: str, handler: Callable[[PatternMatch], Awaitable[None]]
     ) -> "PatternPairAggregator":
         """Register a handler for when a pattern pair is matched.
 
         The handler will be called whenever a complete match for the
-        specified pattern ID is found in the text.
+        specified type is found in the text.
 
         Args:
-            pattern_id: ID of the pattern pair to match.
+            type: The type of the pattern pair to trigger the handler.
             handler: Async function to call when pattern is matched.
                      The function should accept a PatternMatch object.
 
         Returns:
             Self for method chaining.
         """
-        self._handlers[pattern_id] = handler
+        self._handlers[type] = handler
         return self
 
-    async def _process_complete_patterns(self, text: str) -> Tuple[str, bool]:
+    async def _process_complete_patterns(self, text: str) -> Tuple[List[PatternMatch], str]:
         """Process all complete pattern pairs in the text.
 
         Searches for all complete pattern pairs in the text, calls the
@@ -137,19 +228,19 @@ class PatternPairAggregator(BaseTextAggregator):
             text: The text to process for pattern matches.
 
         Returns:
-            Tuple of (processed_text, was_modified) where:
+            Tuple of (all_matches, processed_text) where:
 
-            - processed_text is the text after processing patterns
-            - was_modified indicates whether any changes were made
+            - all_matches is a list of all pattern matches found. Note: There really should only ever be 1.
+            - processed_text is the text after processing patterns. If no patterns are found, it will be the same as input text.
         """
+        all_matches = []
         processed_text = text
-        modified = False
 
-        for pattern_id, pattern_info in self._patterns.items():
+        for type, pattern_info in self._patterns.items():
             # Escape special regex characters in the patterns
             start = re.escape(pattern_info["start"])
             end = re.escape(pattern_info["end"])
-            remove_match = pattern_info["remove_match"]
+            action = pattern_info["action"]
 
             # Create regex to match from start pattern to end pattern
             # The .*? is non-greedy to handle nested patterns
@@ -165,24 +256,25 @@ class PatternPairAggregator(BaseTextAggregator):
 
                 # Create pattern match object
                 pattern_match = PatternMatch(
-                    pattern_id=pattern_id, full_match=full_match, content=content
+                    content=content.strip(), type=type, full_match=full_match
                 )
 
                 # Call the appropriate handler if registered
-                if pattern_id in self._handlers:
+                if type in self._handlers:
                     try:
-                        await self._handlers[pattern_id](pattern_match)
+                        await self._handlers[type](pattern_match)
                     except Exception as e:
-                        logger.error(f"Error in pattern handler for {pattern_id}: {e}")
+                        logger.error(f"Error in pattern handler for {type}: {e}")
 
                 # Remove the pattern from the text if configured
-                if remove_match:
+                if action == MatchAction.REMOVE:
                     processed_text = processed_text.replace(full_match, "", 1)
-                    modified = True
+                else:
+                    all_matches.append(pattern_match)
 
-        return processed_text, modified
+        return all_matches, processed_text
 
-    def _has_incomplete_patterns(self, text: str) -> bool:
+    def _match_start_of_pattern(self, text: str) -> Optional[Tuple[int, dict]]:
         """Check if text contains incomplete pattern pairs.
 
         Determines whether the text contains any start patterns without
@@ -192,9 +284,10 @@ class PatternPairAggregator(BaseTextAggregator):
             text: The text to check for incomplete patterns.
 
         Returns:
-            True if there are incomplete patterns, False otherwise.
+            A tuple of (start_index, pattern_info) if an incomplete pattern is found,
+            or None if no patterns are found or all patterns are complete.
         """
-        for pattern_id, pattern_info in self._patterns.items():
+        for type, pattern_info in self._patterns.items():
             start = pattern_info["start"]
             end = pattern_info["end"]
 
@@ -203,12 +296,16 @@ class PatternPairAggregator(BaseTextAggregator):
             end_count = text.count(end)
 
             # If there are more starts than ends, we have incomplete patterns
+            # Again, this is written generically but there only ever should
+            # be one pattern active at a time, so the counts should be 0 or 1.
+            # Which is why we base the return on the first found.
             if start_count > end_count:
-                return True
+                start_index = text.find(start)
+                return [start_index, pattern_info]
 
-        return False
+        return None
 
-    async def aggregate(self, text: str) -> Optional[str]:
+    async def aggregate(self, text: str) -> Optional[PatternMatch]:
         """Aggregate text and process pattern pairs.
 
         This method adds the new text to the buffer, processes any complete pattern
@@ -227,16 +324,36 @@ class PatternPairAggregator(BaseTextAggregator):
         self._text += text
 
         # Process any complete patterns in the buffer
-        processed_text, modified = await self._process_complete_patterns(self._text)
+        patterns, processed_text = await self._process_complete_patterns(self._text)
 
-        # Only update the buffer if modifications were made
-        if modified:
-            self._text = processed_text
+        self._text = processed_text
+
+        if len(patterns) > 0:
+            if len(patterns) > 1:
+                logger.warning(
+                    f"Multiple patterns matched: {[p.type for p in patterns]}. Only the first pattern will be returned."
+                )
+            # If the pattern found is set to be aggregated, return it
+            action = self._patterns[patterns[0].type].get("action", MatchAction.REMOVE)
+            if action == MatchAction.AGGREGATE:
+                self._text = ""
+                return patterns[0]
 
         # Check if we have incomplete patterns
-        if self._has_incomplete_patterns(self._text):
-            # Still waiting for complete patterns
-            return None
+        pattern_start = self._match_start_of_pattern(self._text)
+        if pattern_start is not None:
+            # If the start pattern is at the beginning or should not be separately aggregated, return None
+            if (
+                pattern_start[0] == 0
+                or pattern_start[1].get("action", MatchAction.REMOVE) != MatchAction.AGGREGATE
+            ):
+                return None
+            # Otherwise, strip the text up to the start pattern and return it
+            result = self._text[: pattern_start[0]]
+            self._text = self._text[pattern_start[0] :]
+            return PatternMatch(
+                content=result.strip(), type=AggregationType.SENTENCE, full_match=result
+            )
 
         # Find sentence boundary if no incomplete patterns
         eos_marker = match_endofsentence(self._text)
@@ -244,7 +361,9 @@ class PatternPairAggregator(BaseTextAggregator):
             # Extract text up to the sentence boundary
             result = self._text[:eos_marker]
             self._text = self._text[eos_marker:]
-            return result
+            return PatternMatch(
+                content=result.strip(), type=AggregationType.SENTENCE, full_match=result
+            )
 
         # No complete sentence found yet
         return None

src/pipecat/utils/text/simple_text_aggregator.py

@@ -14,7 +14,7 @@ text processing scenarios.
 from typing import Optional
 
 from pipecat.utils.string import match_endofsentence
-from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
+from pipecat.utils.text.base_text_aggregator import Aggregation, AggregationType, BaseTextAggregator
 
 
 class SimpleTextAggregator(BaseTextAggregator):
@@ -33,15 +33,15 @@ class SimpleTextAggregator(BaseTextAggregator):
         self._text = ""
 
     @property
-    def text(self) -> str:
+    def text(self) -> Aggregation:
         """Get the currently aggregated text.
 
         Returns:
             The text that has been accumulated in the buffer.
         """
-        return self._text
+        return Aggregation(text=self._text.strip(), type=AggregationType.SENTENCE)
 
-    async def aggregate(self, text: str) -> Optional[str]:
+    async def aggregate(self, text: str) -> Optional[Aggregation]:
         """Aggregate text and return completed sentences.
 
         Adds the new text to the buffer and checks for end-of-sentence markers.
@@ -64,7 +64,9 @@ class SimpleTextAggregator(BaseTextAggregator):
             result = self._text[:eos_end_marker]
             self._text = self._text[eos_end_marker:]
 
-        return result
+        if result:
+            return Aggregation(text=result.strip(), type=AggregationType.SENTENCE)
+        return None
 
     async def handle_interruption(self):
         """Handle interruptions by clearing the text buffer.

src/pipecat/utils/text/skip_tags_aggregator.py

@@ -14,7 +14,7 @@ as a unit regardless of internal punctuation.
 from typing import Optional, Sequence
 
 from pipecat.utils.string import StartEndTags, match_endofsentence, parse_start_end_tags
-from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
+from pipecat.utils.text.base_text_aggregator import Aggregation, AggregationType, BaseTextAggregator
 
 
 class SkipTagsAggregator(BaseTextAggregator):
@@ -43,15 +43,15 @@ class SkipTagsAggregator(BaseTextAggregator):
         self._current_tag_index: int = 0
 
     @property
-    def text(self) -> str:
+    def text(self) -> Aggregation:
         """Get the currently buffered text.
 
         Returns:
             The current text buffer content that hasn't been processed yet.
         """
-        return self._text
+        return Aggregation(text=self._text.strip(), type=AggregationType.SENTENCE)
 
-    async def aggregate(self, text: str) -> Optional[str]:
+    async def aggregate(self, text: str) -> Optional[Aggregation]:
         """Aggregate text while respecting tag boundaries.
 
         This method adds the new text to the buffer, processes any complete
@@ -63,8 +63,9 @@ class SkipTagsAggregator(BaseTextAggregator):
             text: New text to add to the buffer.
 
         Returns:
-            Processed text up to a sentence boundary (when not within tags),
-            or None if more text is needed to complete a sentence or close tags.
+            An Aggregation object containing text up to a sentence boundary and
+            marked as SENTENCE type or None if more text is needed to complete a
+            sentence or close tags.
         """
         # Add new text to buffer
         self._text += text
@@ -80,7 +81,7 @@ class SkipTagsAggregator(BaseTextAggregator):
                 # Extract text up to the sentence boundary
                 result = self._text[:eos_marker]
                 self._text = self._text[eos_marker:]
-                return result
+                return Aggregation(text=result.strip(), type=AggregationType.SENTENCE)
 
         # No complete sentence found yet
         return None

src/pipecat/utils/tracing/service_decorators.py

@@ -23,6 +23,8 @@ if TYPE_CHECKING:
     from opentelemetry import context as context_api
     from opentelemetry import trace
 
+from pipecat.processors.aggregators.llm_context import NOT_GIVEN, LLMContext
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.utils.tracing.service_attributes import (
     add_gemini_live_span_attributes,
     add_llm_span_attributes,
@@ -413,43 +415,47 @@ def traced_llm(func: Optional[Callable] = None, *, name: Optional[str] = None) -
                             # Replace push_frame to capture output
                             self.push_frame = traced_push_frame
 
-                            # Detect if we're using Google's service
-                            is_google_service = "google" in service_class_name.lower()
-
-                            # Try to get messages based on service type
+                            # Get messages for logging
+                            # For OpenAILLMContext: use context's own get_messages_for_logging() method
+                            # For LLMContext: use adapter's get_messages_for_logging() which returns
+                            # messages in provider's native format with sensitive data sanitized
                             messages = None
                             serialized_messages = None
 
-                            # TODO: Revisit once we unify the messages across services
-                            if is_google_service:
-                                # Handle Google service specifically
-                                if hasattr(context, "get_messages_for_logging"):
-                                    messages = context.get_messages_for_logging()
-                            else:
-                                # Handle other services like OpenAI
-                                if hasattr(context, "get_messages"):
-                                    messages = context.get_messages()
-                                elif hasattr(context, "messages"):
-                                    messages = context.messages
+                            if isinstance(context, OpenAILLMContext):
+                                # OpenAILLMContext and subclasses have their own method
+                                messages = context.get_messages_for_logging()
+                            elif isinstance(context, LLMContext):
+                                # Universal LLMContext - use adapter for provider-native format
+                                if hasattr(self, "get_llm_adapter"):
+                                    adapter = self.get_llm_adapter()
+                                    messages = adapter.get_messages_for_logging(context)
 
                             # Serialize messages if available
                             if messages:
-                                try:
-                                    serialized_messages = json.dumps(messages)
-                                except Exception as e:
-                                    serialized_messages = f"Error serializing messages: {str(e)}"
+                                serialized_messages = json.dumps(messages)
 
-                            # Get tools, system message, etc. based on the service type
-                            tools = getattr(context, "tools", None)
+                            # Get tools
+                            # For OpenAILLMContext: tools may need adapter conversion if set
+                            # For LLMContext: use adapter's from_standard_tools() to convert ToolsSchema
+                            tools = None
                             serialized_tools = None
                             tool_count = 0
 
-                            if tools:
-                                try:
-                                    serialized_tools = json.dumps(tools)
-                                    tool_count = len(tools) if isinstance(tools, list) else 1
-                                except Exception as e:
-                                    serialized_tools = f"Error serializing tools: {str(e)}"
+                            if isinstance(context, OpenAILLMContext):
+                                # OpenAILLMContext: tools property handles adapter conversion internally
+                                tools = context.tools
+                            elif isinstance(context, LLMContext):
+                                # Universal LLMContext - use adapter to convert ToolsSchema
+                                if hasattr(self, "get_llm_adapter") and hasattr(context, "tools"):
+                                    adapter = self.get_llm_adapter()
+                                    tools = adapter.from_standard_tools(context.tools)
+
+                            # Serialize and count tools if available
+                            # Check if tools is not None and not NOT_GIVEN
+                            if tools is not None and tools is not NOT_GIVEN:
+                                serialized_tools = json.dumps(tools)
+                                tool_count = len(tools) if isinstance(tools, list) else 1
 
                             # Handle system message for different services
                             system_message = None