Merge branch 'main' into piper-tts

# Conflicts:
#	test-requirements.txt

src/pipecat/__init__.py

@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from importlib.metadata import version
+
+from loguru import logger
+
+__version__ = version("pipecat-ai")
+
+logger.info(f"ᓚᘏᗢ Pipecat {__version__} ᓚᘏᗢ")

src/pipecat/adapters/base_llm_adapter.py

@@ -0,0 +1,22 @@
+from abc import ABC, abstractmethod
+from typing import Any, List, Union, cast
+
+from loguru import logger
+
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+
+
+class BaseLLMAdapter(ABC):
+    @abstractmethod
+    def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Any]:
+        """Converts tools to the provider's format."""
+        pass
+
+    def from_standard_tools(self, tools: Any) -> List[Any]:
+        if isinstance(tools, ToolsSchema):
+            logger.debug(f"Retrieving the tools using the adapter: {type(self)}")
+            return self.to_provider_tools_format(tools)
+        # Fallback to return the same tools in case they are not in a standard format
+        return tools
+
+    # TODO: we can move the logic to also handle the Messages here

src/pipecat/adapters/schemas/function_schema.py

@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from typing import Any, Dict, List
+
+
+class FunctionSchema:
+    def __init__(
+        self, name: str, description: str, properties: Dict[str, Any], required: List[str]
+    ) -> None:
+        """Standardized function schema representation.
+
+        :param name: Name of the function.
+        :param description: Description of the function.
+        :param properties: Dictionary defining properties types and descriptions.
+        :param required: List of required parameters.
+        """
+        self._name = name
+        self._description = description
+        self._properties = properties
+        self._required = required
+
+    def to_default_dict(self) -> Dict[str, Any]:
+        """Converts the function schema to a dictionary.
+
+        :return: Dictionary representation of the function schema.
+        """
+        return {
+            "name": self._name,
+            "description": self._description,
+            "parameters": {
+                "type": "object",
+                "properties": self._properties,
+                "required": self._required,
+            },
+        }
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def description(self) -> str:
+        return self._description
+
+    @property
+    def properties(self) -> Dict[str, Any]:
+        return self._properties
+
+    @property
+    def required(self) -> List[str]:
+        return self._required

src/pipecat/adapters/schemas/tools_schema.py

@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from enum import Enum
+from typing import Any, Dict, List
+
+from pipecat.adapters.schemas.function_schema import FunctionSchema
+
+
+class AdapterType(Enum):
+    GEMINI = "gemini"  # that is the only service where we are able to add custom tools for now
+
+
+class ToolsSchema:
+    def __init__(
+        self,
+        standard_tools: List[FunctionSchema],
+        custom_tools: Dict[AdapterType, List[Dict[str, Any]]] = None,
+    ) -> None:
+        """
+        A schema for tools that includes both standardized function schemas
+        and custom tools that do not follow the FunctionSchema format.
+
+        :param standard_tools: List of tools following FunctionSchema.
+        :param custom_tools: List of tools in a custom format (e.g., search_tool).
+        """
+        self._standard_tools = standard_tools
+        self._custom_tools = custom_tools
+
+    @property
+    def standard_tools(self) -> List[FunctionSchema]:
+        return self._standard_tools
+
+    @property
+    def custom_tools(self) -> Dict[AdapterType, List[Dict[str, Any]]]:
+        return self._custom_tools
+
+    @custom_tools.setter
+    def custom_tools(self, value: Dict[AdapterType, List[Dict[str, Any]]]) -> None:
+        self._custom_tools = value

src/pipecat/adapters/services/anthropic_adapter.py

@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from typing import Any, Dict, List, Union
+
+from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
+from pipecat.adapters.schemas.function_schema import FunctionSchema
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+
+
+class AnthropicLLMAdapter(BaseLLMAdapter):
+    @staticmethod
+    def _to_anthropic_function_format(function: FunctionSchema) -> Dict[str, Any]:
+        return {
+            "name": function.name,
+            "description": function.description,
+            "input_schema": {
+                "type": "object",
+                "properties": function.properties,
+                "required": function.required,
+            },
+        }
+
+    def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
+        """Converts function schemas to Anthropic's function-calling format.
+
+        :return: Anthropic formatted function call definition.
+        """
+
+        functions_schema = tools_schema.standard_tools
+        return [self._to_anthropic_function_format(func) for func in functions_schema]

src/pipecat/adapters/services/gemini_adapter.py

@@ -0,0 +1,28 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from typing import Any, Dict, List, Union
+
+from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
+from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
+
+
+class GeminiLLMAdapter(BaseLLMAdapter):
+    def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
+        """Converts function schemas to Gemini's function-calling format.
+
+        :return: Gemini formatted function call definition.
+        """
+
+        functions_schema = tools_schema.standard_tools
+        formatted_standard_tools = [
+            {"function_declarations": [func.to_default_dict() for func in functions_schema]}
+        ]
+        custom_gemini_tools = []
+        if tools_schema.custom_tools:
+            custom_gemini_tools = tools_schema.custom_tools.get(AdapterType.GEMINI, [])
+
+        return formatted_standard_tools + custom_gemini_tools

src/pipecat/adapters/services/open_ai_adapter.py

@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+from typing import List
+
+from openai.types.chat import ChatCompletionToolParam
+
+from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+
+
+class OpenAILLMAdapter(BaseLLMAdapter):
+    def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[ChatCompletionToolParam]:
+        """Converts function schemas to OpenAI's function-calling format.
+
+        :return: OpenAI formatted function call definition.
+        """
+        functions_schema = tools_schema.standard_tools
+        return [
+            ChatCompletionToolParam(type="function", function=func.to_default_dict())
+            for func in functions_schema
+        ]

src/pipecat/adapters/services/open_ai_realtime_adapter.py

@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+from typing import Any, Dict, List, Union
+
+from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
+from pipecat.adapters.schemas.function_schema import FunctionSchema
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+
+
+class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
+    @staticmethod
+    def _to_openai_realtime_function_format(function: FunctionSchema) -> Dict[str, Any]:
+        return {
+            "type": "function",
+            "name": function.name,
+            "description": function.description,
+            "parameters": {
+                "type": "object",
+                "properties": function.properties,
+                "required": function.required,
+            },
+        }
+
+    def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
+        """Converts function schemas to Openai Realtime function-calling format.
+
+        :return: Openai Realtime formatted function call definition.
+        """
+
+        functions_schema = tools_schema.standard_tools
+        return [self._to_openai_realtime_function_format(func) for func in functions_schema]

src/pipecat/audio/filters/krisp_filter.py

@@ -20,6 +20,22 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+class KrispProcessorManager:
+    """
+    Ensures that only one KrispAudioProcessor instance exists for the entire program.
+    """
+
+    _krisp_instance = None
+
+    @classmethod
+    def get_processor(cls, sample_rate: int, sample_type: str, channels: int, model_path: str):
+        if cls._krisp_instance is None:
+            cls._krisp_instance = KrispAudioProcessor(
+                sample_rate, sample_type, channels, model_path
+            )
+        return cls._krisp_instance
+
+
 class KrispFilter(BaseAudioFilter):
     def __init__(
         self, sample_type: str = "PCM_16", channels: int = 1, model_path: str = None
@@ -48,7 +64,7 @@ class KrispFilter(BaseAudioFilter):
 
     async def start(self, sample_rate: int):
         self._sample_rate = sample_rate
-        self._krisp_processor = KrispAudioProcessor(
+        self._krisp_processor = KrispProcessorManager.get_processor(
             self._sample_rate, self._sample_type, self._channels, self._model_path
         )
 

src/pipecat/audio/utils.py

@@ -18,23 +18,6 @@ def create_default_resampler(**kwargs) -> BaseAudioResampler:
     return SOXRAudioResampler(**kwargs)
 
 
-def resample_audio(audio: bytes, original_rate: int, target_rate: int) -> bytes:
-    import warnings
-
-    with warnings.catch_warnings():
-        warnings.simplefilter("always")
-        warnings.warn(
-            "'resample_audio()' is deprecated, use 'create_default_resampler()' instead.",
-            DeprecationWarning,
-        )
-
-    if original_rate == target_rate:
-        return audio
-    audio_data = np.frombuffer(audio, dtype=np.int16)
-    resampled_audio = soxr.resample(audio_data, original_rate, target_rate)
-    return resampled_audio.astype(np.int16).tobytes()
-
-
 def mix_audio(audio1: bytes, audio2: bytes) -> bytes:
     data1 = np.frombuffer(audio1, dtype=np.int16)
     data2 = np.frombuffer(audio2, dtype=np.int16)

src/pipecat/audio/vad/silero.py

@@ -5,6 +5,7 @@
 #
 
 import time
+from typing import Optional
 
 import numpy as np
 from loguru import logger
@@ -104,11 +105,8 @@ class SileroOnnxModel:
 
 
 class SileroVADAnalyzer(VADAnalyzer):
-    def __init__(self, *, sample_rate: int = 16000, params: VADParams = VADParams()):
-        super().__init__(sample_rate=sample_rate, num_channels=1, params=params)
-
-        if sample_rate != 16000 and sample_rate != 8000:
-            raise ValueError("Silero VAD sample rate needs to be 16000 or 8000")
+    def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams = VADParams()):
+        super().__init__(sample_rate=sample_rate, params=params)
 
         logger.debug("Loading Silero VAD model...")
 
@@ -138,6 +136,12 @@ class SileroVADAnalyzer(VADAnalyzer):
     # VADAnalyzer
     #
 
+    def set_sample_rate(self, sample_rate: int):
+        if sample_rate != 16000 and sample_rate != 8000:
+            raise ValueError("Silero VAD sample rate needs to be 16000 or 8000")
+
+        super().set_sample_rate(sample_rate)
+
     def num_frames_required(self) -> int:
         return 512 if self.sample_rate == 16000 else 256
 

src/pipecat/audio/vad/vad_analyzer.py

@@ -4,8 +4,9 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-from abc import abstractmethod
+from abc import ABC, abstractmethod
 from enum import Enum
+from typing import Optional
 
 from loguru import logger
 from pydantic import BaseModel
@@ -32,12 +33,12 @@ class VADParams(BaseModel):
     min_volume: float = VAD_MIN_VOLUME
 
 
-class VADAnalyzer:
-    def __init__(self, *, sample_rate: int, num_channels: int, params: VADParams):
-        self._sample_rate = sample_rate
-        self._num_channels = num_channels
-
-        self.set_params(params)
+class VADAnalyzer(ABC):
+    def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams):
+        self._init_sample_rate = sample_rate
+        self._sample_rate = 0
+        self._params = params
+        self._num_channels = 1
 
         self._vad_buffer = b""
 
@@ -65,13 +66,17 @@ class VADAnalyzer:
     def voice_confidence(self, buffer) -> float:
         pass
 
+    def set_sample_rate(self, sample_rate: int):
+        self._sample_rate = self._init_sample_rate or sample_rate
+        self.set_params(self._params)
+
     def set_params(self, params: VADParams):
         logger.info(f"Setting VAD params to: {params}")
         self._params = params
         self._vad_frames = self.num_frames_required()
         self._vad_frames_num_bytes = self._vad_frames * self._num_channels * 2
 
-        vad_frames_per_sec = self._vad_frames / self._sample_rate
+        vad_frames_per_sec = self._vad_frames / self.sample_rate
 
         self._vad_start_frames = round(self._params.start_secs / vad_frames_per_sec)
         self._vad_stop_frames = round(self._params.stop_secs / vad_frames_per_sec)
@@ -80,7 +85,7 @@ class VADAnalyzer:
         self._vad_state: VADState = VADState.QUIET
 
     def _get_smoothed_volume(self, audio: bytes) -> float:
-        volume = calculate_audio_volume(audio, self._sample_rate)
+        volume = calculate_audio_volume(audio, self.sample_rate)
         return exp_smoothing(volume, self._prev_volume, self._smoothing_factor)
 
     def analyze_audio(self, buffer) -> VADState:

src/pipecat/frames/frames.py

@@ -6,13 +6,24 @@
 
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import TYPE_CHECKING, Any, Awaitable, Callable, List, Literal, Mapping, Optional, Tuple
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Mapping,
+    Optional,
+    Tuple,
+)
 
 from pipecat.audio.vad.vad_analyzer import VADParams
 from pipecat.clocks.base_clock import BaseClock
 from pipecat.metrics.metrics import MetricsData
 from pipecat.transcriptions.language import Language
-from pipecat.utils.asyncio import TaskManager
+from pipecat.utils.asyncio import BaseTaskManager
 from pipecat.utils.time import nanoseconds_to_str
 from pipecat.utils.utils import obj_count, obj_id
 
@@ -37,7 +48,7 @@ class KeypadEntry(str, Enum):
     STAR = "*"
 
 
-def format_pts(pts: int | None):
+def format_pts(pts: Optional[int]):
     return nanoseconds_to_str(pts) if pts else None
 
 
@@ -48,13 +59,13 @@ class Frame:
     id: int = field(init=False)
     name: str = field(init=False)
     pts: Optional[int] = field(init=False)
-    metadata: dict = field(init=False)
+    metadata: Dict[str, Any] = field(init=False)
 
     def __post_init__(self):
         self.id: int = obj_id()
         self.name: str = f"{self.__class__.__name__}#{obj_count(self)}"
         self.pts: Optional[int] = None
-        self.metadata: dict = {}
+        self.metadata: Dict[str, Any] = {}
 
     def __str__(self):
         return self.name
@@ -115,7 +126,7 @@ class ImageRawFrame:
 
     image: bytes
     size: Tuple[int, int]
-    format: str | None
+    format: Optional[str]
 
 
 #
@@ -165,7 +176,7 @@ class URLImageRawFrame(OutputImageRawFrame):
 
     """
 
-    url: str | None
+    url: Optional[str]
 
     def __str__(self):
         pts = format_pts(self.pts)
@@ -224,7 +235,7 @@ class TranscriptionFrame(TextFrame):
 
     user_id: str
     timestamp: str
-    language: Language | None = None
+    language: Optional[Language] = None
 
     def __str__(self):
         return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})"
@@ -239,7 +250,7 @@ class InterimTranscriptionFrame(TextFrame):
     text: str
     user_id: str
     timestamp: str
-    language: Language | None = None
+    language: Optional[Language] = None
 
     def __str__(self):
         return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})"
@@ -261,7 +272,7 @@ class TranscriptionMessage:
 
     role: Literal["user", "assistant"]
     content: str
-    timestamp: str | None = None
+    timestamp: Optional[str] = None
 
 
 @dataclass
@@ -352,6 +363,13 @@ class LLMSetToolsFrame(DataFrame):
     tools: List[dict]
 
 
+@dataclass
+class LLMSetToolChoiceFrame(DataFrame):
+    """A frame containing a tool choice for an LLM to use for function calling."""
+
+    tool_choice: Literal["none", "auto", "required"] | dict
+
+
 @dataclass
 class LLMEnablePromptCachingFrame(DataFrame):
     """A frame to enable/disable prompt caching in certain LLMs."""
@@ -373,7 +391,7 @@ class FunctionCallResultFrame(DataFrame):
 
     function_name: str
     tool_call_id: str
-    arguments: str
+    arguments: Any
     result: Any
     properties: Optional[FunctionCallResultProperties] = None
 
@@ -427,12 +445,14 @@ class StartFrame(SystemFrame):
     """This is the first frame that should be pushed down a pipeline."""
 
     clock: BaseClock
-    task_manager: TaskManager
+    task_manager: BaseTaskManager
+    audio_in_sample_rate: int = 16000
+    audio_out_sample_rate: int = 24000
     allow_interruptions: bool = False
     enable_metrics: bool = False
     enable_usage_metrics: bool = False
-    report_only_initial_ttfb: bool = False
     observer: Optional["BaseObserver"] = None
+    report_only_initial_ttfb: bool = False
 
 
 @dataclass
@@ -500,9 +520,9 @@ class CancelTaskFrame(SystemFrame):
 
 @dataclass
 class StopTaskFrame(SystemFrame):
-    """Indicates that a pipeline task should be stopped but that the pipeline
-    processors should be kept in a running state. This is normally queued from
-    the pipeline task.
+    """This is used to notify the pipeline task that it should be stopped as
+    soon as possible (flushing all the queued frames) but that the pipeline
+    processors should be kept in a running state.
 
     """
 
@@ -552,6 +572,24 @@ class UserStoppedSpeakingFrame(SystemFrame):
     pass
 
 
+@dataclass
+class EmulateUserStartedSpeakingFrame(SystemFrame):
+    """Emitted by internal processors upstream to emulate VAD behavior when a
+    user starts speaking.
+    """
+
+    pass
+
+
+@dataclass
+class EmulateUserStoppedSpeakingFrame(SystemFrame):
+    """Emitted by internal processors upstream to emulate VAD behavior when a
+    user stops speaking.
+    """
+
+    pass
+
+
 @dataclass
 class BotInterruptionFrame(SystemFrame):
     """Emitted by when the bot should be interrupted. This will mainly cause the
@@ -602,7 +640,23 @@ class FunctionCallInProgressFrame(SystemFrame):
 
     function_name: str
     tool_call_id: str
-    arguments: str
+    arguments: Any
+    cancel_on_interruption: bool = False
+
+
+@dataclass
+class FunctionCallCancelFrame(SystemFrame):
+    """A frame to signal a function call has been cancelled."""
+
+    function_name: str
+    tool_call_id: str
+
+
+@dataclass
+class STTMuteFrame(SystemFrame):
+    """System frame to mute/unmute the STT service."""
+
+    mute: bool
 
 
 @dataclass
@@ -615,13 +669,19 @@ class TransportMessageUrgentFrame(SystemFrame):
 
 @dataclass
 class UserImageRequestFrame(SystemFrame):
-    """A frame user to request an image from the given user."""
+    """A frame to request an image from the given user. The frame might be
+    generated by a function call in which case the corresponding fields will be
+    properly set.
+
+    """
 
     user_id: str
     context: Optional[Any] = None
+    function_name: Optional[str] = None
+    tool_call_id: Optional[str] = None
 
     def __str__(self):
-        return f"{self.name}, user: {self.user_id}"
+        return f"{self.name}(user: {self.user_id}, function: {self.function_name}, request: {self.tool_call_id})"
 
 
 @dataclass
@@ -651,17 +711,18 @@ class UserImageRawFrame(InputImageRawFrame):
     """An image associated to a user."""
 
     user_id: str
+    request: Optional[UserImageRequestFrame] = None
 
     def __str__(self):
         pts = format_pts(self.pts)
-        return f"{self.name}(pts: {pts}, user: {self.user_id}, size: {self.size}, format: {self.format})"
+        return f"{self.name}(pts: {pts}, user: {self.user_id}, size: {self.size}, format: {self.format}, request: {self.request})"
 
 
 @dataclass
 class VisionImageRawFrame(InputImageRawFrame):
     """An image with an associated text to ask for a description of it."""
 
-    text: str | None
+    text: Optional[str]
 
     def __str__(self):
         pts = format_pts(self.pts)
@@ -686,6 +747,17 @@ class EndFrame(ControlFrame):
     pass
 
 
+@dataclass
+class StopFrame(ControlFrame):
+    """Indicates that a pipeline should be stopped but that the pipeline
+    processors should be kept in a running state. This is normally queued from
+    the pipeline task.
+
+    """
+
+    pass
+
+
 @dataclass
 class LLMFullResponseStartFrame(ControlFrame):
     """Used to indicate the beginning of an LLM response. Following by one or
@@ -739,13 +811,6 @@ class TTSUpdateSettingsFrame(ServiceUpdateSettingsFrame):
     pass
 
 
-@dataclass
-class STTMuteFrame(ControlFrame):
-    """Control frame to mute/unmute the STT service."""
-
-    mute: bool
-
-
 @dataclass
 class STTUpdateSettingsFrame(ServiceUpdateSettingsFrame):
     pass

src/pipecat/observers/loggers/llm_log_observer.py

@@ -0,0 +1,85 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from loguru import logger
+
+from pipecat.frames.frames import (
+    Frame,
+    FunctionCallInProgressFrame,
+    FunctionCallResultFrame,
+    LLMFullResponseEndFrame,
+    LLMFullResponseStartFrame,
+    LLMMessagesFrame,
+    LLMTextFrame,
+)
+from pipecat.observers.base_observer import BaseObserver
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.ai_services import LLMService
+
+
+class LLMLogObserver(BaseObserver):
+    """Observer to log LLM activity to the console.
+
+    Logs all frame instances (only from/to LLM service) of:
+
+    - LLMFullResponseStartFrame
+    - LLMFullResponseEndFrame
+    - LLMTextFrame
+    - FunctionCallInProgressFrame
+    - LLMMessagesFrame
+    - OpenAILLMContextFrame
+
+    This allows you to track when the LLM starts responding, what it generates,
+    and when it finishes.
+
+    """
+
+    async def on_push_frame(
+        self,
+        src: FrameProcessor,
+        dst: FrameProcessor,
+        frame: Frame,
+        direction: FrameDirection,
+        timestamp: int,
+    ):
+        if not isinstance(src, LLMService) and not isinstance(dst, LLMService):
+            return
+
+        time_sec = timestamp / 1_000_000_000
+
+        arrow = "→"
+
+        # Log LLM start/end frames (output)
+        if isinstance(frame, (LLMFullResponseStartFrame, LLMFullResponseEndFrame)):
+            event = "START" if isinstance(frame, LLMFullResponseStartFrame) else "END"
+            logger.debug(f"🧠 {src} {arrow} LLM {event} RESPONSE at {time_sec:.2f}s")
+        # Log all LLMTextFrames (output)
+        elif isinstance(frame, LLMTextFrame):
+            logger.debug(f"🧠 {src} {arrow} LLM GENERATING: {frame.text!r} at {time_sec:.2f}s")
+        # Log function calling (output)
+        elif (
+            isinstance(frame, FunctionCallInProgressFrame)
+            and direction != FrameDirection.DOWNSTREAM
+        ):
+            logger.debug(
+                f"🧠 {src} {arrow} LLM FUNCTION CALL ({frame.tool_call_id}): {frame.function_name!r}({frame.arguments}) at {time_sec:.2f}s"
+            )
+        # Log LLMMessagesFrame (input)
+        elif isinstance(frame, LLMMessagesFrame):
+            logger.debug(
+                f"🧠 {arrow} {dst} LLM MESSAGES FRAME: {frame.messages} at {time_sec:.2f}s"
+            )
+        # Log OpenAILLMContextFrame (input)
+        elif isinstance(frame, OpenAILLMContextFrame):
+            logger.debug(
+                f"🧠 {arrow} {dst} LLM CONTEXT FRAME: {frame.context.messages} at {time_sec:.2f}s"
+            )
+        # Log function call result (input)
+        elif isinstance(frame, FunctionCallResultFrame):
+            logger.debug(
+                f"🧠 {arrow} {src} LLM FUNCTION CALL RESULT ({frame.tool_call_id}): {frame.result} at {time_sec:.2f}s"
+            )

src/pipecat/observers/loggers/transcription_log_observer.py

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from loguru import logger
+
+from pipecat.frames.frames import (
+    Frame,
+    InterimTranscriptionFrame,
+    TranscriptionFrame,
+)
+from pipecat.observers.base_observer import BaseObserver
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.ai_services import STTService
+
+
+class TranscriptionLogObserver(BaseObserver):
+    """Observer to log transcription activity to the console.
+
+    Logs all frame instances (only from STT service) of:
+
+    - TranscriptionFrame
+    - InterimTranscriptionFrame
+
+    This allows you to track when the LLM starts responding, what it generates,
+    and when it finishes.
+
+    """
+
+    async def on_push_frame(
+        self,
+        src: FrameProcessor,
+        dst: FrameProcessor,
+        frame: Frame,
+        direction: FrameDirection,
+        timestamp: int,
+    ):
+        if not isinstance(src, STTService):
+            return
+
+        time_sec = timestamp / 1_000_000_000
+
+        arrow = "→"
+
+        if isinstance(frame, TranscriptionFrame):
+            logger.debug(
+                f"💬 {src} {arrow} TRANSCRIPTION: {frame.text!r} from {frame.user_id!r} at {time_sec:.2f}s"
+            )
+        elif isinstance(frame, InterimTranscriptionFrame):
+            logger.debug(
+                f"💬 {src} {arrow} INTERIM TRANSCRIPTION: {frame.text!r} from {frame.user_id!r} at {time_sec:.2f}s"
+            )

src/pipecat/pipeline/base_task.py

@@ -5,25 +5,14 @@
 #
 
 import asyncio
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from typing import AsyncIterable, Iterable
 
 from pipecat.frames.frames import Frame
+from pipecat.utils.base_object import BaseObject
 
 
-class BaseTask(ABC):
-    @property
-    @abstractmethod
-    def id(self) -> int:
-        """Returns the unique indetifier for this task."""
-        pass
-
-    @property
-    @abstractmethod
-    def name(self) -> str:
-        """Returns the name of this task."""
-        pass
-
+class BaseTask(BaseObject):
     @abstractmethod
     def set_event_loop(self, loop: asyncio.AbstractEventLoop):
         """Sets the event loop that this task will run on."""

src/pipecat/pipeline/parallel_pipeline.py

@@ -81,6 +81,8 @@ class ParallelPipeline(BasePipeline):
         self._seen_ids = set()
         self._endframe_counter: Dict[int, int] = {}
 
+        self._up_task = None
+        self._down_task = None
         self._up_queue = asyncio.Queue()
         self._down_queue = asyncio.Queue()
 
@@ -150,19 +152,30 @@ class ParallelPipeline(BasePipeline):
         await self._create_tasks()
 
     async def _stop(self):
-        # The up task doesn't receive an EndFrame, so we just cancel it.
-        await self.cancel_task(self._up_task)
-        # The down tasks waits for the last EndFrame sent by the internal
-        # pipelines.
-        await self._down_task
+        if self._up_task:
+            # The up task doesn't receive an EndFrame, so we just cancel it.
+            await self.cancel_task(self._up_task)
+            self._up_task = None
+
+        if self._down_task:
+            # The down tasks waits for the last EndFrame sent by the internal
+            # pipelines.
+            await self._down_task
+            self._down_task = None
 
     async def _cancel(self):
-        await self.cancel_task(self._up_task)
-        await self.cancel_task(self._down_task)
+        if self._up_task:
+            await self.cancel_task(self._up_task)
+            self._up_task = None
+        if self._down_task:
+            await self.cancel_task(self._down_task)
+            self._down_task = None
 
     async def _create_tasks(self):
-        self._up_task = self.create_task(self._process_up_queue())
-        self._down_task = self.create_task(self._process_down_queue())
+        if not self._up_task:
+            self._up_task = self.create_task(self._process_up_queue())
+        if not self._down_task:
+            self._down_task = self.create_task(self._process_down_queue())
 
     async def _drain_queues(self):
         while not self._up_queue.empty:

src/pipecat/pipeline/runner.py

@@ -12,20 +12,19 @@ from typing import Optional
 from loguru import logger
 
 from pipecat.pipeline.task import PipelineTask
-from pipecat.utils.utils import obj_count, obj_id
+from pipecat.utils.base_object import BaseObject
 
 
-class PipelineRunner:
+class PipelineRunner(BaseObject):
     def __init__(
         self,
         *,
-        name: str | None = None,
+        name: Optional[str] = None,
         handle_sigint: bool = True,
         force_gc: bool = False,
         loop: Optional[asyncio.AbstractEventLoop] = None,
     ):
-        self.id: int = obj_id()
-        self.name: str = name or f"{self.__class__.__name__}#{obj_count(self)}"
+        super().__init__(name=name)
 
         self._tasks = {}
         self._sig_task = None
@@ -41,12 +40,18 @@ class PipelineRunner:
         task.set_event_loop(self._loop)
         await task.run()
         del self._tasks[task.name]
+
+        # Cleanup base object.
+        await self.cleanup()
+
         # If we are cancelling through a signal, make sure we wait for it so
         # everything gets cleaned up nicely.
         if self._sig_task:
             await self._sig_task
+
         if self._force_gc:
             self._gc_collect()
+
         logger.debug(f"Runner {self} finished running {task}")
 
     async def stop_when_done(self):
@@ -74,6 +79,3 @@ class PipelineRunner:
         collected = gc.collect()
         logger.debug(f"Garbage collector: collected {collected} objects.")
         logger.debug(f"Garbage collector: uncollectable objects {gc.garbage}")
-
-    def __str__(self):
-        return self.name

src/pipecat/pipeline/task.py

@@ -5,7 +5,8 @@
 #
 
 import asyncio
-from typing import AsyncIterable, Iterable, List
+import time
+from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Tuple, Type
 
 from loguru import logger
 from pydantic import BaseModel, ConfigDict
@@ -13,6 +14,7 @@ from pydantic import BaseModel, ConfigDict
 from pipecat.clocks.base_clock import BaseClock
 from pipecat.clocks.system_clock import SystemClock
 from pipecat.frames.frames import (
+    BotSpeakingFrame,
     CancelFrame,
     CancelTaskFrame,
     EndFrame,
@@ -20,8 +22,10 @@ from pipecat.frames.frames import (
     ErrorFrame,
     Frame,
     HeartbeatFrame,
+    LLMFullResponseEndFrame,
     MetricsFrame,
     StartFrame,
+    StopFrame,
     StopTaskFrame,
 )
 from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData
@@ -30,32 +34,55 @@ from pipecat.pipeline.base_pipeline import BasePipeline
 from pipecat.pipeline.base_task import BaseTask
 from pipecat.pipeline.task_observer import TaskObserver
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.utils.asyncio import TaskManager
-from pipecat.utils.utils import obj_count, obj_id
+from pipecat.utils.asyncio import BaseTaskManager, TaskManager
 
 HEARTBEAT_SECONDS = 1.0
 HEARTBEAT_MONITOR_SECONDS = HEARTBEAT_SECONDS * 5
 
 
 class PipelineParams(BaseModel):
+    """Configuration parameters for pipeline execution.
+
+    Attributes:
+        allow_interruptions: Whether to allow pipeline interruptions.
+        audio_in_sample_rate: Input audio sample rate in Hz.
+        audio_out_sample_rate: Output audio sample rate in Hz.
+        enable_heartbeats: Whether to enable heartbeat monitoring.
+        enable_metrics: Whether to enable metrics collection.
+        enable_usage_metrics: Whether to enable usage metrics.
+        heartbeats_period_secs: Period between heartbeats in seconds.
+        observers: List of observers for monitoring pipeline execution.
+        report_only_initial_ttfb: Whether to report only initial time to first byte.
+        send_initial_empty_metrics: Whether to send initial empty metrics.
+        start_metadata: Additional metadata for pipeline start.
+    """
+
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     allow_interruptions: bool = False
+    audio_in_sample_rate: int = 16000
+    audio_out_sample_rate: int = 24000
     enable_heartbeats: bool = False
     enable_metrics: bool = False
     enable_usage_metrics: bool = False
-    send_initial_empty_metrics: bool = True
-    report_only_initial_ttfb: bool = False
-    observers: List[BaseObserver] = []
     heartbeats_period_secs: float = HEARTBEAT_SECONDS
+    observers: List[BaseObserver] = []
+    report_only_initial_ttfb: bool = False
+    send_initial_empty_metrics: bool = True
+    start_metadata: Dict[str, Any] = {}
 
 
 class PipelineTaskSource(FrameProcessor):
-    """This is the source processor that is linked at the beginning of the
+    """Source processor for pipeline tasks that handles frame routing.
+
+    This is the source processor that is linked at the beginning of the
     pipeline given to the pipeline task. It allows us to easily push frames
     downstream to the pipeline and also receive upstream frames coming from the
     pipeline.
 
+    Args:
+        up_queue: Queue for upstream frame processing.
+
     """
 
     def __init__(self, up_queue: asyncio.Queue, **kwargs):
@@ -73,10 +100,14 @@ class PipelineTaskSource(FrameProcessor):
 
 
 class PipelineTaskSink(FrameProcessor):
-    """This is the sink processor that is linked at the end of the pipeline
+    """Sink processor for pipeline tasks that handles final frame processing.
+
+    This is the sink processor that is linked at the end of the pipeline
     given to the pipeline task. It allows us to receive downstream frames and
     act on them, for example, waiting to receive an EndFrame.
 
+    Args:
+        down_queue: Queue for downstream frame processing.
     """
 
     def __init__(self, down_queue: asyncio.Queue, **kwargs):
@@ -89,18 +120,80 @@ class PipelineTaskSink(FrameProcessor):
 
 
 class PipelineTask(BaseTask):
+    """Manages the execution of a pipeline, handling frame processing and task lifecycle.
+
+    It has a couple of event handlers `on_frame_reached_upstream` and
+    `on_frame_reached_downstream` that are called when upstream frames or
+    downstream frames reach both ends of pipeline. By default, the events
+    handlers will not be called unless some filters are set using
+    `set_reached_upstream_filter` and `set_reached_downstream_filter`.
+
+       @task.event_handler("on_frame_reached_upstream")
+       async def on_frame_reached_upstream(task, frame):
+           ...
+
+       @task.event_handler("on_frame_reached_downstream")
+       async def on_frame_reached_downstream(task, frame):
+           ...
+
+    It also has an event handler that detects when the pipeline is idle. By
+    default, a pipeline is idle if no `BotSpeakingFrame` or
+    `LLMFullResponseEndFrame` are received within `idle_timeout_secs`.
+
+       @task.event_handler("on_idle_timeout")
+       async def on_idle_timeout(task):
+           ...
+
+    Args:
+        pipeline: The pipeline to execute.
+        params: Configuration parameters for the pipeline.
+        observers: List of observers for monitoring pipeline execution.
+        clock: Clock implementation for timing operations.
+        check_dangling_tasks: Whether to check for processors' tasks finishing properly.
+        idle_timeout_secs: Timeout (in seconds) to consider pipeline idle or
+            None. If a pipeline is idle the pipeline task will be cancelled
+            automatically.
+        idle_timeout_frames: A tuple with the frames that should trigger an idle
+            timeout if not received withing `idle_timeout_seconds`.
+        cancel_on_idle_timeout: Whether the pipeline task should be cancelled if
+            the idle timeout is reached.
+
+    """
+
     def __init__(
         self,
         pipeline: BasePipeline,
+        *,
         params: PipelineParams = PipelineParams(),
+        observers: List[BaseObserver] = [],
         clock: BaseClock = SystemClock(),
+        task_manager: Optional[BaseTaskManager] = None,
+        check_dangling_tasks: bool = True,
+        idle_timeout_secs: Optional[float] = 300,
+        idle_timeout_frames: Tuple[Type[Frame], ...] = (
+            BotSpeakingFrame,
+            LLMFullResponseEndFrame,
+        ),
+        cancel_on_idle_timeout: bool = True,
     ):
-        self._id: int = obj_id()
-        self._name: str = f"{self.__class__.__name__}#{obj_count(self)}"
-
+        super().__init__()
         self._pipeline = pipeline
         self._clock = clock
         self._params = params
+        self._check_dangling_tasks = check_dangling_tasks
+        self._idle_timeout_secs = idle_timeout_secs
+        self._idle_timeout_frames = idle_timeout_frames
+        self._cancel_on_idle_timeout = cancel_on_idle_timeout
+        if self._params.observers:
+            import warnings
+
+            with warnings.catch_warnings():
+                warnings.simplefilter("always")
+                warnings.warn(
+                    "Field 'observers' is deprecated, use the 'observers' parameter instead.",
+                    DeprecationWarning,
+                )
+            observers = self._params.observers
         self._finished = False
 
         # This queue receives frames coming from the pipeline upstream.
@@ -112,33 +205,69 @@ class PipelineTask(BaseTask):
         # This is the heartbeat queue. When a heartbeat frame is received in the
         # down queue we add it to the heartbeat queue for processing.
         self._heartbeat_queue = asyncio.Queue()
-        # This event is used to indicate an EndFrame has been received in the
-        # down queue.
-        self._endframe_event = asyncio.Event()
+        # This is the idle queue. When frames are received downstream they are
+        # put in the queue. If no frame is received the pipeline is considered
+        # idle.
+        self._idle_queue = asyncio.Queue()
+        # This event is used to indicate a finalize frame (e.g. EndFrame,
+        # StopFrame) has been received in the down queue.
+        self._pipeline_end_event = asyncio.Event()
 
+        # This is a source processor that we connect to the provided
+        # pipeline. This source processor allows up to receive and react to
+        # upstream frames.
         self._source = PipelineTaskSource(self._up_queue)
         self._source.link(pipeline)
 
+        # This is a sink processor that we connect to the provided
+        # pipeline. This sink processor allows up to receive and react to
+        # downstream frames.
         self._sink = PipelineTaskSink(self._down_queue)
         pipeline.link(self._sink)
 
-        self._task_manager = TaskManager()
+        # This task maneger will handle all the asyncio tasks created by this
+        # PipelineTask and its frame processors.
+        self._task_manager = task_manager or TaskManager()
 
-        self._observer = TaskObserver(observers=params.observers, task_manager=self._task_manager)
+        # The task observer acts as a proxy to the provided observers. This way,
+        # we only need to pass a single observer (using the StartFrame) which
+        # then just acts as a proxy.
+        self._observer = TaskObserver(observers=observers, task_manager=self._task_manager)
+
+        # These events can be used to check which frames make it to the source
+        # or sink processors. Instead of calling the event handlers for every
+        # frame the user needs to specify which events they are interested
+        # in. This is mainly for efficiency reason because each event handler
+        # creates a task and most likely you only care about one or two frame
+        # types.
+        self._reached_upstream_types: Tuple[Type[Frame], ...] = ()
+        self._reached_downstream_types: Tuple[Type[Frame], ...] = ()
+        self._register_event_handler("on_frame_reached_upstream")
+        self._register_event_handler("on_frame_reached_downstream")
+        self._register_event_handler("on_idle_timeout")
 
     @property
-    def id(self) -> int:
-        """Returns the unique indetifier for this task."""
-        return self._id
-
-    @property
-    def name(self) -> str:
-        """Returns the name of this task."""
-        return self._name
+    def params(self) -> PipelineParams:
+        """Returns the pipeline parameters of this task."""
+        return self._params
 
     def set_event_loop(self, loop: asyncio.AbstractEventLoop):
         self._task_manager.set_event_loop(loop)
 
+    def set_reached_upstream_filter(self, types: Tuple[Type[Frame], ...]):
+        """Sets which frames will be checked before calling the
+        on_frame_reached_upstream event handler.
+
+        """
+        self._reached_upstream_types = types
+
+    def set_reached_downstream_filter(self, types: Tuple[Type[Frame], ...]):
+        """Sets which frames will be checked before calling the
+        on_frame_reached_downstream event handler.
+
+        """
+        self._reached_downstream_types = types
+
     def has_finished(self) -> bool:
         """Indicates whether the tasks has finished. That is, all processors
         have stopped.
@@ -155,9 +284,7 @@ class PipelineTask(BaseTask):
         await self.queue_frame(EndFrame())
 
     async def cancel(self):
-        """
-        Stops the running pipeline immediately.
-        """
+        """Stops the running pipeline immediately."""
         logger.debug(f"Canceling pipeline task {self}")
         # Make sure everything is cleaned up downstream. This is sent
         # out-of-band from the main streaming task which is what we want since
@@ -167,14 +294,15 @@ class PipelineTask(BaseTask):
         await self._task_manager.cancel_task(self._process_push_task)
 
     async def run(self):
-        """
-        Starts running the given pipeline.
-        """
+        """Starts and manages the pipeline execution until completion or cancellation."""
         if self.has_finished():
             return
+        cleanup_pipeline = True
         try:
             push_task = await self._create_tasks()
             await self._task_manager.wait_for_task(push_task)
+            # We have already cleaned up the pipeline inside the task.
+            cleanup_pipeline = False
         except asyncio.CancelledError:
             # We are awaiting on the push task and it might be cancelled
             # (e.g. Ctrl-C). This means we will get a CancelledError here as
@@ -182,19 +310,24 @@ class PipelineTask(BaseTask):
             # awaiting a task.
             pass
         await self._cancel_tasks()
-        await self._cleanup()
-        self._print_dangling_tasks()
+        await self._cleanup(cleanup_pipeline)
+        if self._check_dangling_tasks:
+            self._print_dangling_tasks()
         self._finished = True
 
     async def queue_frame(self, frame: Frame):
-        """
-        Queue a frame to be pushed down the pipeline.
+        """Queue a single frame to be pushed down the pipeline.
+
+        Args:
+            frame: The frame to be processed.
         """
         await self._push_queue.put(frame)
 
     async def queue_frames(self, frames: Iterable[Frame] | AsyncIterable[Frame]):
-        """
-        Queues multiple frames to be pushed down the pipeline.
+        """Queues multiple frames to be pushed down the pipeline.
+
+        Args:
+            frames: An iterable or async iterable of frames to be processed.
         """
         if isinstance(frames, AsyncIterable):
             async for frame in frames:
@@ -227,19 +360,30 @@ class PipelineTask(BaseTask):
                 self._heartbeat_monitor_handler(), f"{self}::_heartbeat_monitor_handler"
             )
 
+    def _maybe_start_idle_task(self):
+        if self._idle_timeout_secs:
+            self._idle_monitor_task = self._task_manager.create_task(
+                self._idle_monitor_handler(), f"{self}::_idle_monitor_handler"
+            )
+
     async def _cancel_tasks(self):
-        await self._maybe_cancel_heartbeat_tasks()
+        await self._observer.stop()
 
         await self._task_manager.cancel_task(self._process_up_task)
         await self._task_manager.cancel_task(self._process_down_task)
 
-        await self._observer.stop()
+        await self._maybe_cancel_heartbeat_tasks()
+        await self._maybe_cancel_idle_task()
 
     async def _maybe_cancel_heartbeat_tasks(self):
         if self._params.enable_heartbeats:
             await self._task_manager.cancel_task(self._heartbeat_push_task)
             await self._task_manager.cancel_task(self._heartbeat_monitor_task)
 
+    async def _maybe_cancel_idle_task(self):
+        if self._idle_timeout_secs:
+            await self._task_manager.cancel_task(self._idle_monitor_task)
+
     def _initial_metrics_frame(self) -> MetricsFrame:
         processors = self._pipeline.processors_with_metrics()
         data = []
@@ -248,52 +392,59 @@ class PipelineTask(BaseTask):
             data.append(ProcessingMetricsData(processor=p.name, value=0.0))
         return MetricsFrame(data=data)
 
-    async def _wait_for_endframe(self):
-        await self._endframe_event.wait()
-        self._endframe_event.clear()
+    async def _wait_for_pipeline_end(self):
+        await self._pipeline_end_event.wait()
+        self._pipeline_end_event.clear()
 
-    async def _cleanup(self):
+    async def _cleanup(self, cleanup_pipeline: bool):
+        # Cleanup base object.
+        await self.cleanup()
+
+        # Cleanup pipeline processors.
         await self._source.cleanup()
-        await self._pipeline.cleanup()
+        if cleanup_pipeline:
+            await self._pipeline.cleanup()
         await self._sink.cleanup()
 
     async def _process_push_queue(self):
         """This is the task that runs the pipeline for the first time by sending
         a StartFrame and by pushing any other frames queued by the user. It runs
-        until the tasks is canceled or stopped (e.g. with an EndFrame).
+        until the tasks is cancelled or stopped (e.g. with an EndFrame).
 
         """
         self._clock.start()
 
         self._maybe_start_heartbeat_tasks()
+        self._maybe_start_idle_task()
 
         start_frame = StartFrame(
             clock=self._clock,
             task_manager=self._task_manager,
             allow_interruptions=self._params.allow_interruptions,
+            audio_in_sample_rate=self._params.audio_in_sample_rate,
+            audio_out_sample_rate=self._params.audio_out_sample_rate,
             enable_metrics=self._params.enable_metrics,
             enable_usage_metrics=self._params.enable_usage_metrics,
-            report_only_initial_ttfb=self._params.report_only_initial_ttfb,
             observer=self._observer,
+            report_only_initial_ttfb=self._params.report_only_initial_ttfb,
         )
+        start_frame.metadata = self._params.start_metadata
         await self._source.queue_frame(start_frame, FrameDirection.DOWNSTREAM)
 
         if self._params.enable_metrics and self._params.send_initial_empty_metrics:
             await self._source.queue_frame(self._initial_metrics_frame(), FrameDirection.DOWNSTREAM)
 
         running = True
-        should_cleanup = True
+        cleanup_pipeline = True
         while running:
             frame = await self._push_queue.get()
             await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM)
-            if isinstance(frame, EndFrame):
-                await self._wait_for_endframe()
-            running = not isinstance(frame, (CancelFrame, EndFrame, StopTaskFrame))
-            should_cleanup = not isinstance(frame, StopTaskFrame)
+            if isinstance(frame, (EndFrame, StopFrame)):
+                await self._wait_for_pipeline_end()
+            running = not isinstance(frame, (CancelFrame, EndFrame, StopFrame))
+            cleanup_pipeline = not isinstance(frame, StopFrame)
             self._push_queue.task_done()
-        # Cleanup only if we need to.
-        if should_cleanup:
-            await self._cleanup()
+        await self._cleanup(cleanup_pipeline)
 
     async def _process_up_queue(self):
         """This is the task that processes frames coming upstream from the
@@ -304,6 +455,10 @@ class PipelineTask(BaseTask):
         """
         while True:
             frame = await self._up_queue.get()
+
+            if isinstance(frame, self._reached_upstream_types):
+                await self._call_event_handler("on_frame_reached_upstream", frame)
+
             if isinstance(frame, EndTaskFrame):
                 # Tell the task we should end nicely.
                 await self.queue_frame(EndFrame())
@@ -311,14 +466,17 @@ class PipelineTask(BaseTask):
                 # Tell the task we should end right away.
                 await self.queue_frame(CancelFrame())
             elif isinstance(frame, StopTaskFrame):
-                await self.queue_frame(StopTaskFrame())
+                # Tell the task we should stop nicely.
+                await self.queue_frame(StopFrame())
             elif isinstance(frame, ErrorFrame):
-                logger.error(f"Error running app: {frame}")
                 if frame.fatal:
+                    logger.error(f"A fatal error occurred: {frame}")
                     # Cancel all tasks downstream.
                     await self.queue_frame(CancelFrame())
                     # Tell the task we should stop.
                     await self.queue_frame(StopTaskFrame())
+                else:
+                    logger.warning(f"Something went wrong: {frame}")
             self._up_queue.task_done()
 
     async def _process_down_queue(self):
@@ -330,16 +488,22 @@ class PipelineTask(BaseTask):
         """
         while True:
             frame = await self._down_queue.get()
-            if isinstance(frame, EndFrame):
-                self._endframe_event.set()
+
+            # Queue received frame to the idle queue so we can monitor idle
+            # pipelines.
+            await self._idle_queue.put(frame)
+
+            if isinstance(frame, self._reached_downstream_types):
+                await self._call_event_handler("on_frame_reached_downstream", frame)
+
+            if isinstance(frame, (EndFrame, StopFrame)):
+                self._pipeline_end_event.set()
             elif isinstance(frame, HeartbeatFrame):
                 await self._heartbeat_queue.put(frame)
             self._down_queue.task_done()
 
     async def _heartbeat_push_handler(self):
-        """
-        This tasks pushes a heartbeat frame every heartbeat period.
-        """
+        """This tasks pushes a heartbeat frame every heartbeat period."""
         while True:
             # Don't use `queue_frame()` because if an EndFrame is queued the
             # task will just stop waiting for the pipeline to finish not
@@ -366,10 +530,49 @@ class PipelineTask(BaseTask):
                     f"{self}: heartbeat frame not received for more than {wait_time} seconds"
                 )
 
+    async def _idle_monitor_handler(self):
+        """This tasks monitors activity in the pipeline. If no frames are
+        received (heartbeats don't count) the pipeline is considered idle.
+
+        """
+        running = True
+        last_frame_time = 0
+        while running:
+            try:
+                frame = await asyncio.wait_for(
+                    self._idle_queue.get(), timeout=self._idle_timeout_secs
+                )
+
+                if isinstance(frame, StartFrame) or isinstance(frame, self._idle_timeout_frames):
+                    # If we find a StartFrame or one of the frames that prevents a
+                    # time out we update the time.
+                    last_frame_time = time.time()
+                else:
+                    # If we find any other frame we check if the pipeline is
+                    # idle by checking the last time we received one of the
+                    # valid frames.
+                    diff_time = time.time() - last_frame_time
+                    if diff_time >= self._idle_timeout_secs:
+                        running = await self._idle_timeout_detected()
+
+                self._idle_queue.task_done()
+            except asyncio.TimeoutError:
+                running = await self._idle_timeout_detected()
+
+    async def _idle_timeout_detected(self) -> bool:
+        """Logic for when the pipeline is idle.
+
+        Returns:
+            bool: Whther the pipeline task is being cancelled or not.
+        """
+        await self._call_event_handler("on_idle_timeout")
+        if self._cancel_on_idle_timeout:
+            logger.warning(f"Idle pipeline detected, cancelling pipeline task...")
+            await self.cancel()
+            return False
+        return True
+
     def _print_dangling_tasks(self):
         tasks = [t.get_name() for t in self._task_manager.current_tasks()]
         if tasks:
             logger.warning(f"Dangling tasks detected: {tasks}")
-
-    def __str__(self):
-        return self.name

src/pipecat/pipeline/task_observer.py

@@ -12,8 +12,7 @@ from attr import dataclass
 from pipecat.frames.frames import Frame
 from pipecat.observers.base_observer import BaseObserver
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.utils.asyncio import TaskManager
-from pipecat.utils.utils import obj_count, obj_id
+from pipecat.utils.asyncio import BaseTaskManager
 
 
 @dataclass
@@ -55,21 +54,11 @@ class TaskObserver(BaseObserver):
 
     """
 
-    def __init__(self, *, observers: List[BaseObserver] = [], task_manager: TaskManager):
-        self._id: int = obj_id()
-        self._name: str = f"{self.__class__.__name__}#{obj_count(self)}"
+    def __init__(self, *, observers: List[BaseObserver] = [], task_manager: BaseTaskManager):
         self._observers = observers
         self._task_manager = task_manager
         self._proxies: List[Proxy] = []
 
-    @property
-    def id(self) -> int:
-        return self._id
-
-    @property
-    def name(self) -> str:
-        return self._name
-
     async def start(self):
         """Starts all proxy observer tasks."""
         self._proxies = self._create_proxies(self._observers)
@@ -100,7 +89,7 @@ class TaskObserver(BaseObserver):
             queue = asyncio.Queue()
             task = self._task_manager.create_task(
                 self._proxy_task_handler(queue, observer),
-                f"{self}::{observer.__class__.__name__}::_proxy_task_handler",
+                f"TaskObserver::{observer.__class__.__name__}::_proxy_task_handler",
             )
             proxy = Proxy(queue=queue, task=task, observer=observer)
             proxies.append(proxy)
@@ -112,6 +101,3 @@ class TaskObserver(BaseObserver):
             await observer.on_push_frame(
                 data.src, data.dst, data.frame, data.direction, data.timestamp
             )
-
-    def __str__(self):
-        return self.name

src/pipecat/processors/aggregators/gated_openai_llm_context.py

@@ -21,6 +21,7 @@ class GatedOpenAILLMContextAggregator(FrameProcessor):
         self._notifier = notifier
         self._start_open = start_open
         self._last_context_frame = None
+        self._gate_task = None
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
@@ -41,10 +42,13 @@ class GatedOpenAILLMContextAggregator(FrameProcessor):
             await self.push_frame(frame, direction)
 
     async def _start(self):
-        self._gate_task = self.create_task(self._gate_task_handler())
+        if not self._gate_task:
+            self._gate_task = self.create_task(self._gate_task_handler())
 
     async def _stop(self):
-        await self.cancel_task(self._gate_task)
+        if self._gate_task:
+            await self.cancel_task(self._gate_task)
+            self._gate_task = None
 
     async def _gate_task_handler(self):
         while True:

src/pipecat/processors/aggregators/llm_response.py

@@ -4,20 +4,37 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-from typing import List, Type
+import asyncio
+from abc import abstractmethod
+from typing import Dict, List, Literal, Set
+
+from loguru import logger
 
 from pipecat.frames.frames import (
+    BotStoppedSpeakingFrame,
+    CancelFrame,
+    EmulateUserStartedSpeakingFrame,
+    EmulateUserStoppedSpeakingFrame,
+    EndFrame,
     Frame,
+    FunctionCallCancelFrame,
+    FunctionCallInProgressFrame,
+    FunctionCallResultFrame,
     InterimTranscriptionFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesAppendFrame,
     LLMMessagesFrame,
     LLMMessagesUpdateFrame,
+    LLMSetToolChoiceFrame,
     LLMSetToolsFrame,
+    LLMTextFrame,
+    OpenAILLMContextAssistantTimestampFrame,
+    StartFrame,
     StartInterruptionFrame,
     TextFrame,
     TranscriptionFrame,
+    UserImageRawFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
 )
@@ -26,236 +43,152 @@ from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContextFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-
-
-class LLMResponseAggregator(FrameProcessor):
-    def __init__(
-        self,
-        *,
-        messages: List[dict],
-        role: str,
-        start_frame,
-        end_frame,
-        accumulator_frame: Type[TextFrame],
-        interim_accumulator_frame: Type[TextFrame] | None = None,
-        handle_interruptions: bool = False,
-        expect_stripped_words: bool = True,  # if True, need to add spaces between words
-    ):
-        super().__init__()
-
-        self._messages = messages
-        self._role = role
-        self._start_frame = start_frame
-        self._end_frame = end_frame
-        self._accumulator_frame = accumulator_frame
-        self._interim_accumulator_frame = interim_accumulator_frame
-        self._handle_interruptions = handle_interruptions
-        self._expect_stripped_words = expect_stripped_words
-
-        # Reset our accumulator state.
-        self._reset()
-
-    @property
-    def messages(self):
-        return self._messages
-
-    @property
-    def role(self):
-        return self._role
-
-    #
-    # Frame processor
-    #
-
-    # Use cases implemented:
-    #
-    # S: Start, E: End, T: Transcription, I: Interim, X: Text
-    #
-    #        S E -> None
-    #      S T E -> X
-    #    S I T E -> X
-    #    S I E T -> X
-    #  S I E I T -> X
-    #      S E T -> X
-    #    S E I T -> X
-    #
-    # The following case would not be supported:
-    #
-    #    S I E T1 I T2 -> X
-    #
-    # and T2 would be dropped.
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        send_aggregation = False
-
-        if isinstance(frame, self._start_frame):
-            self._aggregation = ""
-            self._aggregating = True
-            self._seen_start_frame = True
-            self._seen_end_frame = False
-            self._seen_interim_results = False
-            await self.push_frame(frame, direction)
-        elif isinstance(frame, self._end_frame):
-            self._seen_end_frame = True
-            self._seen_start_frame = False
-
-            # We might have received the end frame but we might still be
-            # aggregating (i.e. we have seen interim results but not the final
-            # text).
-            self._aggregating = self._seen_interim_results or len(self._aggregation) == 0
-
-            # Send the aggregation if we are not aggregating anymore (i.e. no
-            # more interim results received).
-            send_aggregation = not self._aggregating
-            await self.push_frame(frame, direction)
-        elif isinstance(frame, self._accumulator_frame):
-            if self._aggregating:
-                if self._expect_stripped_words:
-                    self._aggregation += f" {frame.text}" if self._aggregation else frame.text
-                else:
-                    self._aggregation += frame.text
-                # We have recevied a complete sentence, so if we have seen the
-                # end frame and we were still aggregating, it means we should
-                # send the aggregation.
-                send_aggregation = self._seen_end_frame
-
-            # We just got our final result, so let's reset interim results.
-            self._seen_interim_results = False
-        elif self._interim_accumulator_frame and isinstance(frame, self._interim_accumulator_frame):
-            self._seen_interim_results = True
-        elif self._handle_interruptions and isinstance(frame, StartInterruptionFrame):
-            await self._push_aggregation()
-            # Reset anyways
-            self._reset()
-            await self.push_frame(frame, direction)
-        elif isinstance(frame, LLMMessagesAppendFrame):
-            self._add_messages(frame.messages)
-        elif isinstance(frame, LLMMessagesUpdateFrame):
-            self._set_messages(frame.messages)
-        elif isinstance(frame, LLMSetToolsFrame):
-            self._set_tools(frame.tools)
-        else:
-            await self.push_frame(frame, direction)
-
-        if send_aggregation:
-            await self._push_aggregation()
-
-    async def _push_aggregation(self):
-        if len(self._aggregation) > 0:
-            self._messages.append({"role": self._role, "content": self._aggregation})
-
-            # Reset the aggregation. Reset it before pushing it down, otherwise
-            # if the tasks gets cancelled we won't be able to clear things up.
-            self._aggregation = ""
-
-            frame = LLMMessagesFrame(self._messages)
-            await self.push_frame(frame)
-
-    # TODO-CB: Types
-    def _add_messages(self, messages):
-        self._messages.extend(messages)
-
-    def _set_messages(self, messages):
-        self._reset()
-        self._messages.clear()
-        self._messages.extend(messages)
-
-    def _set_tools(self, tools):
-        # noop in the base class
-        pass
-
-    def _reset(self):
-        self._aggregation = ""
-        self._aggregating = False
-        self._seen_start_frame = False
-        self._seen_end_frame = False
-        self._seen_interim_results = False
-
-
-class LLMAssistantResponseAggregator(LLMResponseAggregator):
-    def __init__(self, messages: List[dict] = []):
-        super().__init__(
-            messages=messages,
-            role="assistant",
-            start_frame=LLMFullResponseStartFrame,
-            end_frame=LLMFullResponseEndFrame,
-            accumulator_frame=TextFrame,
-            handle_interruptions=True,
-        )
-
-
-class LLMUserResponseAggregator(LLMResponseAggregator):
-    def __init__(self, messages: List[dict] = []):
-        super().__init__(
-            messages=messages,
-            role="user",
-            start_frame=UserStartedSpeakingFrame,
-            end_frame=UserStoppedSpeakingFrame,
-            accumulator_frame=TranscriptionFrame,
-            interim_accumulator_frame=InterimTranscriptionFrame,
-        )
+from pipecat.utils.time import time_now_iso8601
 
 
 class LLMFullResponseAggregator(FrameProcessor):
-    """This class aggregates Text frames until it receives a
-    LLMFullResponseEndFrame, then emits the concatenated text as
-    a single text frame.
+    """This is an LLM aggregator that aggregates a full LLM completion. It
+    aggregates LLM text frames (tokens) received between
+    `LLMFullResponseStartFrame` and `LLMFullResponseEndFrame`. Every full
+    completion is returned via the "on_completion" event handler:
 
-    given the following frames:
+       @aggregator.event_handler("on_completion")
+       async def on_completion(
+           aggregator: LLMFullResponseAggregator,
+           completion: str,
+           completed: bool,
+       )
 
-        TextFrame("Hello,")
-        TextFrame(" world.")
-        TextFrame(" I am")
-        TextFrame(" an LLM.")
-        LLMFullResponseEndFrame()]
-
-    this processor will yield nothing for the first 4 frames, then
-
-        TextFrame("Hello, world. I am an LLM.")
-        LLMFullResponseEndFrame()
-
-    when passed the last frame.
-
-    >>> async def print_frames(aggregator, frame):
-    ...     async for frame in aggregator.process_frame(frame):
-    ...         if isinstance(frame, TextFrame):
-    ...             print(frame.text)
-    ...         else:
-    ...             print(frame.__class__.__name__)
-
-    >>> aggregator = LLMFullResponseAggregator()
-    >>> asyncio.run(print_frames(aggregator, TextFrame("Hello,")))
-    >>> asyncio.run(print_frames(aggregator, TextFrame(" world.")))
-    >>> asyncio.run(print_frames(aggregator, TextFrame(" I am")))
-    >>> asyncio.run(print_frames(aggregator, TextFrame(" an LLM.")))
-    >>> asyncio.run(print_frames(aggregator, LLMFullResponseEndFrame()))
-    Hello, world. I am an LLM.
-    LLMFullResponseEndFrame
     """
 
-    def __init__(self):
-        super().__init__()
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
         self._aggregation = ""
+        self._started = False
+
+        self._register_event_handler("on_completion")
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
 
-        if isinstance(frame, TextFrame):
-            self._aggregation += frame.text
-        elif isinstance(frame, LLMFullResponseEndFrame):
-            await self.push_frame(TextFrame(self._aggregation))
-            await self.push_frame(frame)
+        if isinstance(frame, StartInterruptionFrame):
+            await self._call_event_handler("on_completion", self._aggregation, False)
             self._aggregation = ""
-        else:
-            await self.push_frame(frame, direction)
+            self._started = False
+        elif isinstance(frame, LLMFullResponseStartFrame):
+            await self._handle_llm_start(frame)
+        elif isinstance(frame, LLMFullResponseEndFrame):
+            await self._handle_llm_end(frame)
+        elif isinstance(frame, LLMTextFrame):
+            await self._handle_llm_text(frame)
+
+        await self.push_frame(frame, direction)
+
+    async def _handle_llm_start(self, _: LLMFullResponseStartFrame):
+        self._started = True
+
+    async def _handle_llm_end(self, _: LLMFullResponseEndFrame):
+        await self._call_event_handler("on_completion", self._aggregation, True)
+        self._started = False
+        self._aggregation = ""
+
+    async def _handle_llm_text(self, frame: TextFrame):
+        if not self._started:
+            return
+        self._aggregation += frame.text
 
 
-class LLMContextAggregator(LLMResponseAggregator):
-    def __init__(self, *, context: OpenAILLMContext, **kwargs):
+class BaseLLMResponseAggregator(FrameProcessor):
+    """This is the base class for all LLM response aggregators. These
+    aggregators process incoming frames and aggregate content until they are
+    ready to push the aggregation. In the case of a user, an aggregation might
+    be a full transcription received from the STT service.
+
+    The LLM response aggregators also keep a store (e.g. a message list or an
+    LLM context) of the current conversation, that is, it stores the messages
+    said by the user or by the bot.
+
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @property
+    @abstractmethod
+    def messages(self) -> List[dict]:
+        """Returns the messages from the current conversation."""
+        pass
+
+    @property
+    @abstractmethod
+    def role(self) -> str:
+        """Returns the role (e.g. user, assistant...) for this aggregator."""
+        pass
+
+    @abstractmethod
+    def add_messages(self, messages):
+        """Add the given messages to the conversation."""
+        pass
+
+    @abstractmethod
+    def set_messages(self, messages):
+        """Reset the conversation with the given messages."""
+        pass
+
+    @abstractmethod
+    def set_tools(self, tools):
+        """Set LLM tools to be used in the current conversation."""
+        pass
+
+    @abstractmethod
+    def set_tool_choice(self, tool_choice):
+        """Set the tool choice. This should modify the LLM context."""
+        pass
+
+    @abstractmethod
+    def reset(self):
+        """Reset the internals of this aggregator. This should not modify the
+        internal messages."""
+        pass
+
+    @abstractmethod
+    async def handle_aggregation(self, aggregation: str):
+        """Adds the given aggregation to the aggregator. The aggregator can use
+        a simple list of message or a context. It doesn't not push any frames.
+
+        """
+        pass
+
+    @abstractmethod
+    async def push_aggregation(self):
+        """Pushes the current aggregation. For example, iN the case of context
+        aggregation this might push a new context frame.
+
+        """
+        pass
+
+
+class LLMContextResponseAggregator(BaseLLMResponseAggregator):
+    """This is a base LLM aggregator that uses an LLM context to store the
+    conversation. It pushes `OpenAILLMContextFrame` as an aggregation frame.
+
+    """
+
+    def __init__(self, *, context: OpenAILLMContext, role: str, **kwargs):
         super().__init__(**kwargs)
         self._context = context
+        self._role = role
+
+        self._aggregation = ""
+
+    @property
+    def messages(self) -> List[dict]:
+        return self._context.get_messages()
+
+    @property
+    def role(self) -> str:
+        return self._role
 
     @property
     def context(self):
@@ -264,57 +197,397 @@ class LLMContextAggregator(LLMResponseAggregator):
     def get_context_frame(self) -> OpenAILLMContextFrame:
         return OpenAILLMContextFrame(context=self._context)
 
-    async def push_context_frame(self):
+    async def push_context_frame(self, direction: FrameDirection = FrameDirection.DOWNSTREAM):
         frame = self.get_context_frame()
-        await self.push_frame(frame)
+        await self.push_frame(frame, direction)
 
-    # TODO-CB: Types
-    def _add_messages(self, messages):
+    def add_messages(self, messages):
         self._context.add_messages(messages)
 
-    def _set_messages(self, messages):
+    def set_messages(self, messages):
         self._context.set_messages(messages)
 
-    def _set_tools(self, tools: List):
+    def set_tools(self, tools: List):
         self._context.set_tools(tools)
 
-    async def _push_aggregation(self):
+    def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | dict):
+        self._context.set_tool_choice(tool_choice)
+
+    def reset(self):
+        self._aggregation = ""
+
+
+class LLMUserContextAggregator(LLMContextResponseAggregator):
+    """This is a user LLM aggregator that uses an LLM context to store the
+    conversation. It aggregates transcriptions from the STT service and it has
+    logic to handle multiple scenarios where transcriptions are received between
+    VAD events (`UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame`) or
+    even outside or no VAD events at all.
+
+    """
+
+    def __init__(
+        self,
+        context: OpenAILLMContext,
+        aggregation_timeout: float = 1.0,
+        **kwargs,
+    ):
+        super().__init__(context=context, role="user", **kwargs)
+        self._aggregation_timeout = aggregation_timeout
+
+        self._seen_interim_results = False
+        self._user_speaking = False
+        self._emulating_vad = False
+        self._waiting_for_aggregation = False
+
+        self._aggregation_event = asyncio.Event()
+        self._aggregation_task = None
+
+    def reset(self):
+        super().reset()
+        self._seen_interim_results = False
+        self._waiting_for_aggregation = False
+
+    async def handle_aggregation(self, aggregation: str):
+        self._context.add_message({"role": self.role, "content": self._aggregation})
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, StartFrame):
+            # Push StartFrame before start(), because we want StartFrame to be
+            # processed by every processor before any other frame is processed.
+            await self.push_frame(frame, direction)
+            await self._start(frame)
+        elif isinstance(frame, EndFrame):
+            # Push EndFrame before stop(), because stop() waits on the task to
+            # finish and the task finishes when EndFrame is processed.
+            await self.push_frame(frame, direction)
+            await self._stop(frame)
+        elif isinstance(frame, CancelFrame):
+            await self._cancel(frame)
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, UserStartedSpeakingFrame):
+            await self._handle_user_started_speaking(frame)
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            await self._handle_user_stopped_speaking(frame)
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, TranscriptionFrame):
+            await self._handle_transcription(frame)
+        elif isinstance(frame, InterimTranscriptionFrame):
+            await self._handle_interim_transcription(frame)
+        elif isinstance(frame, LLMMessagesAppendFrame):
+            self.add_messages(frame.messages)
+        elif isinstance(frame, LLMMessagesUpdateFrame):
+            self.set_messages(frame.messages)
+        elif isinstance(frame, LLMSetToolsFrame):
+            self.set_tools(frame.tools)
+        elif isinstance(frame, LLMSetToolChoiceFrame):
+            self.set_tool_choice(frame.tool_choice)
+        else:
+            await self.push_frame(frame, direction)
+
+    async def push_aggregation(self):
         if len(self._aggregation) > 0:
-            self._context.add_message({"role": self._role, "content": self._aggregation})
+            await self.handle_aggregation(self._aggregation)
 
             # Reset the aggregation. Reset it before pushing it down, otherwise
             # if the tasks gets cancelled we won't be able to clear things up.
-            self._aggregation = ""
+            self.reset()
 
             frame = OpenAILLMContextFrame(self._context)
             await self.push_frame(frame)
 
-            # Reset our accumulator state.
-            self._reset()
+    async def _start(self, frame: StartFrame):
+        self._create_aggregation_task()
+
+    async def _stop(self, frame: EndFrame):
+        await self._cancel_aggregation_task()
+
+    async def _cancel(self, frame: CancelFrame):
+        await self._cancel_aggregation_task()
+
+    async def _handle_user_started_speaking(self, _: UserStartedSpeakingFrame):
+        self._user_speaking = True
+        self._waiting_for_aggregation = True
+
+    async def _handle_user_stopped_speaking(self, _: UserStoppedSpeakingFrame):
+        self._user_speaking = False
+        # We just stopped speaking. Let's see if there's some aggregation to
+        # push. If the last thing we saw is an interim transcription, let's wait
+        # pushing the aggregation as we will probably get a final transcription.
+        if not self._seen_interim_results:
+            await self.push_aggregation()
+
+    async def _handle_transcription(self, frame: TranscriptionFrame):
+        text = frame.text
+
+        # Make sure we really have some text.
+        if not text.strip():
+            return
+
+        self._aggregation += f" {text}" if self._aggregation else text
+        # We just got a final result, so let's reset interim results.
+        self._seen_interim_results = False
+        # Reset aggregation timer.
+        self._aggregation_event.set()
+
+    async def _handle_interim_transcription(self, _: InterimTranscriptionFrame):
+        self._seen_interim_results = True
+
+    def _create_aggregation_task(self):
+        if not self._aggregation_task:
+            self._aggregation_task = self.create_task(self._aggregation_task_handler())
+
+    async def _cancel_aggregation_task(self):
+        if self._aggregation_task:
+            await self.cancel_task(self._aggregation_task)
+            self._aggregation_task = None
+
+    async def _aggregation_task_handler(self):
+        while True:
+            try:
+                await asyncio.wait_for(self._aggregation_event.wait(), self._aggregation_timeout)
+                await self._maybe_push_bot_interruption()
+            except asyncio.TimeoutError:
+                if not self._user_speaking:
+                    await self.push_aggregation()
+
+                # If we are emulating VAD we still need to send the user stopped
+                # speaking frame.
+                if self._emulating_vad:
+                    await self.push_frame(
+                        EmulateUserStoppedSpeakingFrame(), FrameDirection.UPSTREAM
+                    )
+                    self._emulating_vad = False
+            finally:
+                self._aggregation_event.clear()
+
+    async def _maybe_push_bot_interruption(self):
+        """If the user stopped speaking a while back and we got a transcription
+        frame we might want to interrupt the bot.
+
+        """
+        if not self._user_speaking and not self._waiting_for_aggregation:
+            # If we reach this case we received a transcription but VAD was not
+            # able to detect voice (e.g. when you whisper a short
+            # utterance). So, we need to emulate VAD (i.e. user start/stopped
+            # speaking).
+            await self.push_frame(EmulateUserStartedSpeakingFrame(), FrameDirection.UPSTREAM)
+            self._emulating_vad = True
 
 
-class LLMAssistantContextAggregator(LLMContextAggregator):
-    def __init__(self, context: OpenAILLMContext, *, expect_stripped_words: bool = True):
-        super().__init__(
-            messages=[],
-            context=context,
-            role="assistant",
-            start_frame=LLMFullResponseStartFrame,
-            end_frame=LLMFullResponseEndFrame,
-            accumulator_frame=TextFrame,
-            handle_interruptions=True,
-            expect_stripped_words=expect_stripped_words,
+class LLMAssistantContextAggregator(LLMContextResponseAggregator):
+    """This is an assistant LLM aggregator that uses an LLM context to store the
+    conversation. It aggregates text frames received between
+    `LLMFullResponseStartFrame` and `LLMFullResponseEndFrame`.
+
+    """
+
+    def __init__(self, context: OpenAILLMContext, *, expect_stripped_words: bool = True, **kwargs):
+        super().__init__(context=context, role="assistant", **kwargs)
+        self._expect_stripped_words = expect_stripped_words
+
+        self._started = 0
+        self._function_calls_in_progress: Dict[str, FunctionCallInProgressFrame] = {}
+        self._context_updated_tasks: Set[asyncio.Task] = set()
+
+    async def handle_aggregation(self, aggregation: str):
+        self._context.add_message({"role": "assistant", "content": aggregation})
+
+    async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
+        pass
+
+    async def handle_function_call_result(self, frame: FunctionCallResultFrame):
+        pass
+
+    async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame):
+        pass
+
+    async def handle_user_image_frame(self, frame: UserImageRawFrame):
+        pass
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, StartInterruptionFrame):
+            await self._handle_interruptions(frame)
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, LLMFullResponseStartFrame):
+            await self._handle_llm_start(frame)
+        elif isinstance(frame, LLMFullResponseEndFrame):
+            await self._handle_llm_end(frame)
+        elif isinstance(frame, TextFrame):
+            await self._handle_text(frame)
+        elif isinstance(frame, LLMMessagesAppendFrame):
+            self.add_messages(frame.messages)
+        elif isinstance(frame, LLMMessagesUpdateFrame):
+            self.set_messages(frame.messages)
+        elif isinstance(frame, LLMSetToolsFrame):
+            self.set_tools(frame.tools)
+        elif isinstance(frame, LLMSetToolChoiceFrame):
+            self.set_tool_choice(frame.tool_choice)
+        elif isinstance(frame, FunctionCallInProgressFrame):
+            await self._handle_function_call_in_progress(frame)
+        elif isinstance(frame, FunctionCallResultFrame):
+            await self._handle_function_call_result(frame)
+        elif isinstance(frame, FunctionCallCancelFrame):
+            await self._handle_function_call_cancel(frame)
+        elif isinstance(frame, UserImageRawFrame) and frame.request and frame.request.tool_call_id:
+            await self._handle_user_image_frame(frame)
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            await self.push_aggregation()
+        else:
+            await self.push_frame(frame, direction)
+
+    async def push_aggregation(self):
+        if not self._aggregation:
+            return
+
+        aggregation = self._aggregation.strip()
+        self.reset()
+
+        if aggregation:
+            await self.handle_aggregation(aggregation)
+
+        # Push context frame
+        await self.push_context_frame()
+
+        # Push timestamp frame with current time
+        timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601())
+        await self.push_frame(timestamp_frame)
+
+    async def _handle_interruptions(self, frame: StartInterruptionFrame):
+        await self.push_aggregation()
+        self._started = 0
+        self.reset()
+
+    async def _handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
+        logger.debug(
+            f"{self} FunctionCallInProgressFrame: [{frame.function_name}:{frame.tool_call_id}]"
+        )
+        await self.handle_function_call_in_progress(frame)
+        self._function_calls_in_progress[frame.tool_call_id] = frame
+
+    async def _handle_function_call_result(self, frame: FunctionCallResultFrame):
+        logger.debug(
+            f"{self} FunctionCallResultFrame: [{frame.function_name}:{frame.tool_call_id}]"
+        )
+        if frame.tool_call_id not in self._function_calls_in_progress:
+            logger.warning(
+                f"FunctionCallResultFrame tool_call_id [{frame.tool_call_id}] is not running"
+            )
+            return
+
+        del self._function_calls_in_progress[frame.tool_call_id]
+
+        properties = frame.properties
+
+        await self.handle_function_call_result(frame)
+
+        # Run inference if the function call result requires it.
+        if frame.result:
+            run_llm = False
+
+            if properties and properties.run_llm is not None:
+                # If the tool call result has a run_llm property, use it
+                run_llm = properties.run_llm
+            else:
+                # Default behavior is to run the LLM if there are no function calls in progress
+                run_llm = not bool(self._function_calls_in_progress)
+
+            if run_llm:
+                await self.push_context_frame(FrameDirection.UPSTREAM)
+
+        # Call the `on_context_updated` callback once the function call result
+        # is added to the context. Also, run this in a separate task to make
+        # sure we don't block the pipeline.
+        if properties and properties.on_context_updated:
+            task_name = f"{frame.function_name}:{frame.tool_call_id}:on_context_updated"
+            task = self.create_task(properties.on_context_updated(), task_name)
+            self._context_updated_tasks.add(task)
+            task.add_done_callback(self._context_updated_task_finished)
+
+    async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame):
+        logger.debug(
+            f"{self} FunctionCallCancelFrame: [{frame.function_name}:{frame.tool_call_id}]"
+        )
+        if frame.tool_call_id not in self._function_calls_in_progress:
+            return
+
+        if self._function_calls_in_progress[frame.tool_call_id].cancel_on_interruption:
+            await self.handle_function_call_cancel(frame)
+            del self._function_calls_in_progress[frame.tool_call_id]
+
+    async def _handle_user_image_frame(self, frame: UserImageRawFrame):
+        logger.debug(
+            f"{self} UserImageRawFrame: [{frame.request.function_name}:{frame.request.tool_call_id}]"
         )
 
+        if frame.request.tool_call_id not in self._function_calls_in_progress:
+            logger.warning(
+                f"UserImageRawFrame tool_call_id [{frame.request.tool_call_id}] is not running"
+            )
+            return
 
-class LLMUserContextAggregator(LLMContextAggregator):
-    def __init__(self, context: OpenAILLMContext):
-        super().__init__(
-            messages=[],
-            context=context,
-            role="user",
-            start_frame=UserStartedSpeakingFrame,
-            end_frame=UserStoppedSpeakingFrame,
-            accumulator_frame=TranscriptionFrame,
-            interim_accumulator_frame=InterimTranscriptionFrame,
-        )
+        del self._function_calls_in_progress[frame.request.tool_call_id]
+
+        await self.handle_user_image_frame(frame)
+        await self.push_aggregation()
+        await self.push_context_frame(FrameDirection.UPSTREAM)
+
+    async def _handle_llm_start(self, _: LLMFullResponseStartFrame):
+        self._started += 1
+
+    async def _handle_llm_end(self, _: LLMFullResponseEndFrame):
+        self._started -= 1
+        await self.push_aggregation()
+
+    async def _handle_text(self, frame: TextFrame):
+        if not self._started:
+            return
+
+        if self._expect_stripped_words:
+            self._aggregation += f" {frame.text}" if self._aggregation else frame.text
+        else:
+            self._aggregation += frame.text
+
+    def _context_updated_task_finished(self, task: asyncio.Task):
+        self._context_updated_tasks.discard(task)
+        # The task is finished so this should exit immediately. We need to do
+        # this because otherwise the task manager would report a dangling task
+        # if we don't remove it.
+        asyncio.run_coroutine_threadsafe(self.wait_for_task(task), self.get_event_loop())
+
+
+class LLMUserResponseAggregator(LLMUserContextAggregator):
+    def __init__(self, messages: List[dict] = [], **kwargs):
+        super().__init__(context=OpenAILLMContext(messages), **kwargs)
+
+    async def push_aggregation(self):
+        if len(self._aggregation) > 0:
+            await self.handle_aggregation(self._aggregation)
+
+            # Reset the aggregation. Reset it before pushing it down, otherwise
+            # if the tasks gets cancelled we won't be able to clear things up.
+            self.reset()
+
+            frame = LLMMessagesFrame(self._context.messages)
+            await self.push_frame(frame)
+
+
+class LLMAssistantResponseAggregator(LLMAssistantContextAggregator):
+    def __init__(self, messages: List[dict] = [], **kwargs):
+        super().__init__(context=OpenAILLMContext(messages), **kwargs)
+
+    async def push_aggregation(self):
+        if len(self._aggregation) > 0:
+            await self.handle_aggregation(self._aggregation)
+
+            # Reset the aggregation. Reset it before pushing it down, otherwise
+            # if the tasks gets cancelled we won't be able to clear things up.
+            self.reset()
+
+            frame = LLMMessagesFrame(self._context.messages)
+            await self.push_frame(frame)

src/pipecat/processors/aggregators/openai_llm_context.py

@@ -9,33 +9,21 @@ import copy
 import io
 import json
 from dataclasses import dataclass
-from typing import Any, Awaitable, Callable, List, Optional
+from typing import Any, List, Optional
 
-from loguru import logger
+from openai._types import NOT_GIVEN, NotGiven
+from openai.types.chat import (
+    ChatCompletionMessageParam,
+    ChatCompletionToolChoiceOptionParam,
+    ChatCompletionToolParam,
+)
 from PIL import Image
 
-from pipecat.frames.frames import (
-    AudioRawFrame,
-    Frame,
-    FunctionCallInProgressFrame,
-    FunctionCallResultFrame,
-)
+from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+from pipecat.frames.frames import AudioRawFrame, Frame
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
-try:
-    from openai._types import NOT_GIVEN, NotGiven
-    from openai.types.chat import (
-        ChatCompletionMessageParam,
-        ChatCompletionToolChoiceOptionParam,
-        ChatCompletionToolParam,
-    )
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error(
-        "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable."
-    )
-    raise Exception(f"Missing module: {e}")
-
 # JSON custom encoder to handle bytes arrays so that we can log contexts
 # with images to the console.
 
@@ -51,14 +39,20 @@ class CustomEncoder(json.JSONEncoder):
 class OpenAILLMContext:
     def __init__(
         self,
-        messages: List[ChatCompletionMessageParam] | None = None,
-        tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN,
+        messages: Optional[List[ChatCompletionMessageParam]] = None,
+        tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN,
         tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,
     ):
         self._messages: List[ChatCompletionMessageParam] = messages if messages else []
         self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice
-        self._tools: List[ChatCompletionToolParam] | NotGiven = tools
-        self._user_image_request_context = {}
+        self._tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = tools
+        self._llm_adapter: Optional[BaseLLMAdapter] = None
+
+    def get_llm_adapter(self) -> Optional[BaseLLMAdapter]:
+        return self._llm_adapter
+
+    def set_llm_adapter(self, llm_adapter: BaseLLMAdapter):
+        self._llm_adapter = llm_adapter
 
     @staticmethod
     def from_messages(messages: List[dict]) -> "OpenAILLMContext":
@@ -75,7 +69,9 @@ class OpenAILLMContext:
         return self._messages
 
     @property
-    def tools(self) -> List[ChatCompletionToolParam] | NotGiven:
+    def tools(self) -> List[ChatCompletionToolParam] | NotGiven | List[Any]:
+        if self._llm_adapter:
+            return self._llm_adapter.from_standard_tools(self._tools)
         return self._tools
 
     @property
@@ -160,8 +156,8 @@ class OpenAILLMContext:
     def set_tool_choice(self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven):
         self._tool_choice = tool_choice
 
-    def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN):
-        if tools != NOT_GIVEN and len(tools) == 0:
+    def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN):
+        if tools != NOT_GIVEN and isinstance(tools, list) and len(tools) == 0:
             tools = NOT_GIVEN
         self._tools = tools
 
@@ -184,61 +180,6 @@ class OpenAILLMContext:
         # todo: implement for OpenAI models and others
         pass
 
-    async def call_function(
-        self,
-        f: Callable[
-            [str, str, Any, FrameProcessor, "OpenAILLMContext", Callable[[Any], Awaitable[None]]],
-            Awaitable[None],
-        ],
-        *,
-        function_name: str,
-        tool_call_id: str,
-        arguments: str,
-        llm: FrameProcessor,
-        run_llm: bool = True,
-    ) -> None:
-        logger.info(f"Calling function {function_name} with arguments {arguments}")
-        # Push a SystemFrame downstream. This frame will let our assistant context aggregator
-        # know that we are in the middle of a function call. Some contexts/aggregators may
-        # not need this. But some definitely do (Anthropic, for example).
-        # Also push a SystemFrame upstream for use by other processors, like STTMuteFilter.
-        progress_frame_downstream = FunctionCallInProgressFrame(
-            function_name=function_name,
-            tool_call_id=tool_call_id,
-            arguments=arguments,
-        )
-        progress_frame_upstream = FunctionCallInProgressFrame(
-            function_name=function_name,
-            tool_call_id=tool_call_id,
-            arguments=arguments,
-        )
-
-        # Push frame both downstream and upstream
-        await llm.push_frame(progress_frame_downstream, FrameDirection.DOWNSTREAM)
-        await llm.push_frame(progress_frame_upstream, FrameDirection.UPSTREAM)
-
-        # Define a callback function that pushes a FunctionCallResultFrame upstream & downstream.
-        async def function_call_result_callback(result, *, properties=None):
-            result_frame_downstream = FunctionCallResultFrame(
-                function_name=function_name,
-                tool_call_id=tool_call_id,
-                arguments=arguments,
-                result=result,
-                properties=properties,
-            )
-            result_frame_upstream = FunctionCallResultFrame(
-                function_name=function_name,
-                tool_call_id=tool_call_id,
-                arguments=arguments,
-                result=result,
-                properties=properties,
-            )
-
-            await llm.push_frame(result_frame_downstream, FrameDirection.DOWNSTREAM)
-            await llm.push_frame(result_frame_upstream, FrameDirection.UPSTREAM)
-
-        await f(function_name, tool_call_id, arguments, llm, self, function_call_result_callback)
-
     def create_wav_header(self, sample_rate, num_channels, bits_per_sample, data_size):
         # RIFF chunk descriptor
         header = bytearray()

src/pipecat/processors/aggregators/user_response.py

@@ -4,129 +4,15 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-from pipecat.frames.frames import (
-    Frame,
-    InterimTranscriptionFrame,
-    StartInterruptionFrame,
-    TextFrame,
-    TranscriptionFrame,
-    UserStartedSpeakingFrame,
-    UserStoppedSpeakingFrame,
-)
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.frames.frames import TextFrame
+from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregator
 
 
-class ResponseAggregator(FrameProcessor):
-    """This frame processor aggregates frames between a start and an end frame
-    into complete text frame sentences.
+class UserResponseAggregator(LLMUserResponseAggregator):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
 
-    For example, frame input/output:
-        UserStartedSpeakingFrame() -> None
-        TranscriptionFrame("Hello,") -> None
-        TranscriptionFrame(" world.") -> None
-        UserStoppedSpeakingFrame() -> TextFrame("Hello world.")
-
-    Doctest: FIXME to work with asyncio
-    >>> async def print_frames(aggregator, frame):
-    ...     async for frame in aggregator.process_frame(frame):
-    ...         if isinstance(frame, TextFrame):
-    ...             print(frame.text)
-
-    >>> aggregator = ResponseAggregator(start_frame = UserStartedSpeakingFrame,
-    ...                                 end_frame=UserStoppedSpeakingFrame,
-    ...                                 accumulator_frame=TranscriptionFrame,
-    ...                                 pass_through=False)
-    >>> asyncio.run(print_frames(aggregator, UserStartedSpeakingFrame()))
-    >>> asyncio.run(print_frames(aggregator, TranscriptionFrame("Hello,", 1, 1)))
-    >>> asyncio.run(print_frames(aggregator, TranscriptionFrame("world.",  1, 2)))
-    >>> asyncio.run(print_frames(aggregator, UserStoppedSpeakingFrame()))
-    Hello, world.
-
-    """
-
-    def __init__(
-        self,
-        *,
-        start_frame,
-        end_frame,
-        accumulator_frame: TextFrame,
-        interim_accumulator_frame: TextFrame | None = None,
-    ):
-        super().__init__()
-
-        self._start_frame = start_frame
-        self._end_frame = end_frame
-        self._accumulator_frame = accumulator_frame
-        self._interim_accumulator_frame = interim_accumulator_frame
-
-        # Reset our accumulator state.
-        self._reset()
-
-    #
-    # Frame processor
-    #
-
-    # Use cases implemented:
-    #
-    # S: Start, E: End, T: Transcription, I: Interim, X: Text
-    #
-    #        S E -> None
-    #      S T E -> X
-    #    S I T E -> X
-    #    S I E T -> X
-    #  S I E I T -> X
-    #      S E T -> X
-    #    S E I T -> X
-    #
-    # The following case would not be supported:
-    #
-    #    S I E T1 I T2 -> X
-    #
-    # and T2 would be dropped.
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        send_aggregation = False
-
-        if isinstance(frame, self._start_frame):
-            self._aggregating = True
-            self._seen_start_frame = True
-            self._seen_end_frame = False
-            self._seen_interim_results = False
-            await self.push_frame(frame, direction)
-        elif isinstance(frame, self._end_frame):
-            self._seen_end_frame = True
-            self._seen_start_frame = False
-
-            # We might have received the end frame but we might still be
-            # aggregating (i.e. we have seen interim results but not the final
-            # text).
-            self._aggregating = self._seen_interim_results or len(self._aggregation) == 0
-
-            # Send the aggregation if we are not aggregating anymore (i.e. no
-            # more interim results received).
-            send_aggregation = not self._aggregating
-            await self.push_frame(frame, direction)
-        elif isinstance(frame, self._accumulator_frame):
-            if self._aggregating:
-                self._aggregation += f" {frame.text}"
-                # We have recevied a complete sentence, so if we have seen the
-                # end frame and we were still aggregating, it means we should
-                # send the aggregation.
-                send_aggregation = self._seen_end_frame
-
-            # We just got our final result, so let's reset interim results.
-            self._seen_interim_results = False
-        elif self._interim_accumulator_frame and isinstance(frame, self._interim_accumulator_frame):
-            self._seen_interim_results = True
-        else:
-            await self.push_frame(frame, direction)
-
-        if send_aggregation:
-            await self._push_aggregation()
-
-    async def _push_aggregation(self):
+    async def push_aggregation(self):
         if len(self._aggregation) > 0:
             frame = TextFrame(self._aggregation.strip())
 
@@ -137,21 +23,4 @@ class ResponseAggregator(FrameProcessor):
             await self.push_frame(frame)
 
             # Reset our accumulator state.
-            self._reset()
-
-    def _reset(self):
-        self._aggregation = ""
-        self._aggregating = False
-        self._seen_start_frame = False
-        self._seen_end_frame = False
-        self._seen_interim_results = False
-
-
-class UserResponseAggregator(ResponseAggregator):
-    def __init__(self):
-        super().__init__(
-            start_frame=UserStartedSpeakingFrame,
-            end_frame=UserStoppedSpeakingFrame,
-            accumulator_frame=TranscriptionFrame,
-            interim_accumulator_frame=InterimTranscriptionFrame,
-        )
+            self.reset()

src/pipecat/processors/audio/audio_buffer_processor.py

@@ -5,44 +5,85 @@
 #
 
 import time
+from typing import Optional
 
 from pipecat.audio.utils import create_default_resampler, interleave_stereo_audio, mix_audio
 from pipecat.frames.frames import (
     AudioRawFrame,
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
     CancelFrame,
     EndFrame,
     Frame,
     InputAudioRawFrame,
     OutputAudioRawFrame,
+    StartFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
 
 class AudioBufferProcessor(FrameProcessor):
-    """This processor buffers audio raw frames (input and output). The mixed
-    audio can be obtained by calling `get_audio()` (if `buffer_size` is 0) or by
-    registering an "on_audio_data" event handler. The event handler will be
-    called every time `buffer_size` is reached.
+    """Processes and buffers audio frames from both input (user) and output (bot) sources.
 
-    You can provide the desired output `sample_rate` and incoming audio frames
-    will resampled to match it. Also, you can provide the number of channels, 1
-    for mono and 2 for stereo. With mono audio user and bot audio will be mixed,
-    in the case of stereo the left channel will be used for the user's audio and
-    the right channel for the bot.
+    This processor manages audio buffering and synchronization, providing both merged and
+    track-specific audio access through event handlers. It supports various audio configurations
+    including sample rate conversion and mono/stereo output.
 
+    Events:
+        on_audio_data: Triggered when buffer_size is reached, providing merged audio
+        on_track_audio_data: Triggered when buffer_size is reached, providing separate tracks
+        on_user_turn_audio_data: Triggered when user turn has ended, providing that user turn's audio
+        on_bot_turn_audio_data: Triggered when bot turn has ended, providing that bot turn's audio
+
+    Args:
+        sample_rate (Optional[int]): Desired output sample rate. If None, uses source rate
+        num_channels (int): Number of channels (1 for mono, 2 for stereo). Defaults to 1
+        buffer_size (int): Size of buffer before triggering events. 0 for no buffering
+        user_continuous_stream (bool): Whether user audio is continuous or speech-only
+        enable_turn_audio (bool): Whether turn audio event handlers should be triggered
+
+    Audio handling:
+        - Mono output (num_channels=1): User and bot audio are mixed
+        - Stereo output (num_channels=2): User audio on left, bot audio on right
+        - Automatic resampling of incoming audio to match desired sample_rate
+        - Silence insertion for non-continuous audio streams
+        - Buffer synchronization between user and bot audio
+
+    Note:
+        When user_continuous_stream is False, the processor expects only speech
+        segments and will handle silence insertion between segments automatically.
     """
 
     def __init__(
-        self, *, sample_rate: int = 24000, num_channels: int = 1, buffer_size: int = 0, **kwargs
+        self,
+        *,
+        sample_rate: Optional[int] = None,
+        num_channels: int = 1,
+        buffer_size: int = 0,
+        user_continuous_stream: bool = True,
+        enable_turn_audio: bool = False,
+        **kwargs,
     ):
         super().__init__(**kwargs)
-        self._sample_rate = sample_rate
+        self._init_sample_rate = sample_rate
+        self._sample_rate = 0
+        self._audio_buffer_size_1s = 0
         self._num_channels = num_channels
         self._buffer_size = buffer_size
+        self._user_continuous_stream = user_continuous_stream
+        self._enable_turn_audio = enable_turn_audio
 
         self._user_audio_buffer = bytearray()
         self._bot_audio_buffer = bytearray()
 
+        self._user_speaking = False
+        self._bot_speaking = False
+        self._user_turn_audio_buffer = bytearray()
+        self._bot_turn_audio_buffer = bytearray()
+
+        # Intermittent (non continous user stream variables)
         self._last_user_frame_at = 0
         self._last_bot_frame_at = 0
 
@@ -51,21 +92,47 @@ class AudioBufferProcessor(FrameProcessor):
         self._resampler = create_default_resampler()
 
         self._register_event_handler("on_audio_data")
+        self._register_event_handler("on_track_audio_data")
+        self._register_event_handler("on_user_turn_audio_data")
+        self._register_event_handler("on_bot_turn_audio_data")
 
     @property
     def sample_rate(self) -> int:
+        """Current sample rate of the audio processor.
+
+        Returns:
+            int: The sample rate in Hz
+        """
         return self._sample_rate
 
     @property
     def num_channels(self) -> int:
+        """Number of channels in the audio output.
+
+        Returns:
+            int: Number of channels (1 for mono, 2 for stereo)
+        """
         return self._num_channels
 
     def has_audio(self) -> bool:
+        """Check if both user and bot audio buffers contain data.
+
+        Returns:
+            bool: True if both buffers contain audio data
+        """
         return self._buffer_has_audio(self._user_audio_buffer) and self._buffer_has_audio(
             self._bot_audio_buffer
         )
 
     def merge_audio_buffers(self) -> bytes:
+        """Merge user and bot audio buffers into a single audio stream.
+
+        For mono output, audio is mixed. For stereo output, user audio is placed
+        on the left channel and bot audio on the right channel.
+
+        Returns:
+            bytes: Mixed audio data
+        """
         if self._num_channels == 1:
             return mix_audio(bytes(self._user_audio_buffer), bytes(self._bot_audio_buffer))
         elif self._num_channels == 2:
@@ -76,17 +143,103 @@ class AudioBufferProcessor(FrameProcessor):
             return b""
 
     async def start_recording(self):
+        """Start recording audio from both user and bot.
+
+        Initializes recording state and resets audio buffers.
+        """
         self._recording = True
         self._reset_recording()
 
     async def stop_recording(self):
+        """Stop recording and trigger final audio data handlers.
+
+        Calls audio handlers with any remaining buffered audio before stopping.
+        """
         await self._call_on_audio_data_handler()
         self._recording = False
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming audio frames and manage audio buffers."""
         await super().process_frame(frame, direction)
 
-        if self._recording and isinstance(frame, InputAudioRawFrame):
+        # Update output sample rate if necessary.
+        if isinstance(frame, StartFrame):
+            self._update_sample_rate(frame)
+
+        if self._recording:
+            await self._process_recording(frame)
+            if self._enable_turn_audio:
+                await self._process_turn_recording(frame)
+
+        if isinstance(frame, (CancelFrame, EndFrame)):
+            await self.stop_recording()
+
+        await self.push_frame(frame, direction)
+
+    def _update_sample_rate(self, frame: StartFrame):
+        self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
+        self._audio_buffer_size_1s = self._sample_rate * 2
+
+    async def _process_recording(self, frame: Frame):
+        if self._user_continuous_stream:
+            await self._handle_continuous_stream(frame)
+        else:
+            await self._handle_intermittent_stream(frame)
+
+        if self._buffer_size > 0 and len(self._user_audio_buffer) > self._buffer_size:
+            await self._call_on_audio_data_handler()
+
+    async def _process_turn_recording(self, frame: Frame):
+        if isinstance(frame, UserStartedSpeakingFrame):
+            self._user_speaking = True
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            await self._call_event_handler(
+                "on_user_turn_audio_data", self._user_turn_audio_buffer, self.sample_rate, 1
+            )
+            self._user_speaking = False
+            self._user_turn_audio_buffer = bytearray()
+        elif isinstance(frame, BotStartedSpeakingFrame):
+            self._bot_speaking = True
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            await self._call_event_handler(
+                "on_bot_turn_audio_data", self._bot_turn_audio_buffer, self.sample_rate, 1
+            )
+            self._bot_speaking = False
+            self._bot_turn_audio_buffer = bytearray()
+
+        if isinstance(frame, InputAudioRawFrame):
+            resampled = await self._resample_audio(frame)
+            self._user_turn_audio_buffer += resampled
+            # In the case of the user, we need to keep a short buffer of audio
+            # since VAD notification of when the user starts speaking comes
+            # later.
+            if (
+                not self._user_speaking
+                and len(self._user_turn_audio_buffer) > self._audio_buffer_size_1s
+            ):
+                discarded = len(self._user_turn_audio_buffer) - self._audio_buffer_size_1s
+                self._user_turn_audio_buffer = self._user_turn_audio_buffer[discarded:]
+        elif self._bot_speaking and isinstance(frame, OutputAudioRawFrame):
+            resampled = await self._resample_audio(frame)
+            self._bot_turn_audio_buffer += resampled
+
+    async def _handle_continuous_stream(self, frame: Frame):
+        if isinstance(frame, InputAudioRawFrame):
+            # Add user audio.
+            resampled = await self._resample_audio(frame)
+            self._user_audio_buffer.extend(resampled)
+            # Sync the bot's buffer to the user's buffer by adding silence if needed
+            if len(self._user_audio_buffer) > len(self._bot_audio_buffer):
+                silence_size = len(self._user_audio_buffer) - len(self._bot_audio_buffer)
+                silence = b"\x00" * silence_size
+                self._bot_audio_buffer.extend(silence)
+        elif self._recording and isinstance(frame, OutputAudioRawFrame):
+            # Add bot audio.
+            resampled = await self._resample_audio(frame)
+            self._bot_audio_buffer.extend(resampled)
+
+    async def _handle_intermittent_stream(self, frame: Frame):
+        if isinstance(frame, InputAudioRawFrame):
             # Add silence if we need to.
             silence = self._compute_silence(self._last_user_frame_at)
             self._user_audio_buffer.extend(silence)
@@ -105,22 +258,25 @@ class AudioBufferProcessor(FrameProcessor):
             # Save time of frame so we can compute silence.
             self._last_bot_frame_at = time.time()
 
-        if self._buffer_size > 0 and len(self._user_audio_buffer) > self._buffer_size:
-            await self._call_on_audio_data_handler()
-
-        if isinstance(frame, (CancelFrame, EndFrame)):
-            await self.stop_recording()
-
-        await self.push_frame(frame, direction)
-
     async def _call_on_audio_data_handler(self):
         if not self.has_audio() or not self._recording:
             return
 
+        # Call original handler with merged audio
         merged_audio = self.merge_audio_buffers()
         await self._call_event_handler(
             "on_audio_data", merged_audio, self._sample_rate, self._num_channels
         )
+
+        # Call new handler with separate tracks
+        await self._call_event_handler(
+            "on_track_audio_data",
+            bytes(self._user_audio_buffer),
+            bytes(self._bot_audio_buffer),
+            self._sample_rate,
+            self._num_channels,
+        )
+
         self._reset_audio_buffers()
 
     def _buffer_has_audio(self, buffer: bytearray) -> bool:
@@ -134,16 +290,17 @@ class AudioBufferProcessor(FrameProcessor):
     def _reset_audio_buffers(self):
         self._user_audio_buffer = bytearray()
         self._bot_audio_buffer = bytearray()
+        self._user_turn_audio_buffer = bytearray()
+        self._bot_turn_audio_buffer = bytearray()
 
     async def _resample_audio(self, frame: AudioRawFrame) -> bytes:
         return await self._resampler.resample(frame.audio, frame.sample_rate, self._sample_rate)
 
     def _compute_silence(self, from_time: float) -> bytes:
         quiet_time = time.time() - from_time
-        # We should get audio frames very frequently. We pick 100ms because
-        # that's big enough, but it could be even a bit slower since we usually
-        # do 20ms audio frames.
-        if from_time == 0 or quiet_time < 0.1:
+        # We should get audio frames very frequently. We introduce silence only
+        # if there's a big enough gap of 1s.
+        if from_time == 0 or quiet_time < 1.0:
             return b""
         num_bytes = int(quiet_time * self._sample_rate) * 2
         silence = b"\x00" * num_bytes

src/pipecat/processors/audio/vad/silero.py

@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+from typing import Optional
+
 from loguru import logger
 
 from pipecat.audio.vad.silero import SileroVADAnalyzer
@@ -11,6 +13,7 @@ from pipecat.audio.vad.vad_analyzer import VADParams, VADState
 from pipecat.frames.frames import (
     AudioRawFrame,
     Frame,
+    StartFrame,
     StartInterruptionFrame,
     StopInterruptionFrame,
     UserStartedSpeakingFrame,
@@ -23,7 +26,7 @@ class SileroVAD(FrameProcessor):
     def __init__(
         self,
         *,
-        sample_rate: int = 16000,
+        sample_rate: Optional[int] = None,
         vad_params: VADParams = VADParams(),
         audio_passthrough: bool = False,
     ):
@@ -41,6 +44,9 @@ class SileroVAD(FrameProcessor):
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
 
+        if isinstance(frame, StartFrame):
+            self._vad_analyzer.set_sample_rate(frame.audio_in_sample_rate)
+
         if isinstance(frame, AudioRawFrame):
             await self._analyze_audio(frame)
             if self._audio_passthrough:

src/pipecat/processors/filters/stt_mute_filter.py

@@ -23,6 +23,8 @@ from pipecat.frames.frames import (
     Frame,
     FunctionCallInProgressFrame,
     FunctionCallResultFrame,
+    InputAudioRawFrame,
+    StartFrame,
     StartInterruptionFrame,
     StopInterruptionFrame,
     STTMuteFrame,
@@ -30,23 +32,24 @@ from pipecat.frames.frames import (
     UserStoppedSpeakingFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.ai_services import STTService
 
 
 class STTMuteStrategy(Enum):
     """Strategies determining when STT should be muted.
 
     Attributes:
-        FIRST_SPEECH: Mute only during first bot speech
+        FIRST_SPEECH: Mute only during first detected bot speech
+        MUTE_UNTIL_FIRST_BOT_COMPLETE: Start muted and remain muted until first bot speech completes
         FUNCTION_CALL: Mute during function calls
         ALWAYS: Mute during all bot speech
         CUSTOM: Allow custom logic via callback
     """
 
-    FIRST_SPEECH = "first_speech"  # Mute only during first bot speech
-    FUNCTION_CALL = "function_call"  # Mute during function calls
-    ALWAYS = "always"  # Mute during all bot speech
-    CUSTOM = "custom"  # Allow custom logic via callback
+    FIRST_SPEECH = "first_speech"
+    MUTE_UNTIL_FIRST_BOT_COMPLETE = "mute_until_first_bot_complete"
+    FUNCTION_CALL = "function_call"
+    ALWAYS = "always"
+    CUSTOM = "custom"
 
 
 @dataclass
@@ -57,12 +60,25 @@ class STTMuteConfig:
         strategies: Set of muting strategies to apply
         should_mute_callback: Optional callback for custom muting logic.
             Only required when using STTMuteStrategy.CUSTOM
+
+    Note:
+        MUTE_UNTIL_FIRST_BOT_COMPLETE and FIRST_SPEECH strategies should not be used together
+        as they handle the first bot speech differently.
     """
 
     strategies: set[STTMuteStrategy]
     # Optional callback for custom muting logic
     should_mute_callback: Optional[Callable[["STTMuteFilter"], Awaitable[bool]]] = None
 
+    def __post_init__(self):
+        if (
+            STTMuteStrategy.MUTE_UNTIL_FIRST_BOT_COMPLETE in self.strategies
+            and STTMuteStrategy.FIRST_SPEECH in self.strategies
+        ):
+            raise ValueError(
+                "MUTE_UNTIL_FIRST_BOT_COMPLETE and FIRST_SPEECH strategies should not be used together"
+            )
+
 
 class STTMuteFilter(FrameProcessor):
     """A processor that handles STT muting and interruption control.
@@ -71,28 +87,29 @@ class STTMuteFilter(FrameProcessor):
     feature. When STT is muted, interruptions are automatically disabled.
 
     Args:
-        stt_service: Service handling speech-to-text functionality
         config: Configuration specifying muting strategies
+        stt_service: STT service instance (deprecated, will be removed in future version)
         **kwargs: Additional arguments passed to parent class
     """
 
-    def __init__(self, stt_service: STTService, config: STTMuteConfig, **kwargs):
+    def __init__(self, *, config: STTMuteConfig, **kwargs):
         super().__init__(**kwargs)
-        self._stt_service = stt_service
         self._config = config
         self._first_speech_handled = False
         self._bot_is_speaking = False
         self._function_call_in_progress = False
+        self._is_muted = False  # Initialize as unmuted, will set state on StartFrame if needed
 
     @property
     def is_muted(self) -> bool:
         """Returns whether STT is currently muted."""
-        return self._stt_service.is_muted
+        return self._is_muted
 
     async def _handle_mute_state(self, should_mute: bool):
         """Handles both STT muting and interruption control."""
         if should_mute != self.is_muted:
             logger.debug(f"STT {'muting' if should_mute else 'unmuting'}")
+            self._is_muted = should_mute
             await self.push_frame(STTMuteFrame(mute=should_mute))
 
     async def _should_mute(self) -> bool:
@@ -112,6 +129,10 @@ class STTMuteFilter(FrameProcessor):
                         self._first_speech_handled = True
                         return True
 
+                case STTMuteStrategy.MUTE_UNTIL_FIRST_BOT_COMPLETE:
+                    if not self._first_speech_handled:
+                        return True
+
                 case STTMuteStrategy.CUSTOM:
                     if self._bot_is_speaking and self._config.should_mute_callback:
                         should_mute = await self._config.should_mute_callback(self)
@@ -121,25 +142,31 @@ class STTMuteFilter(FrameProcessor):
         return False
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Processes incoming frames and manages muting state."""
         await super().process_frame(frame, direction)
 
-        """Processes incoming frames and manages muting state."""
-        # Handle function call state changes
-        if isinstance(frame, FunctionCallInProgressFrame):
+        # Determine if we need to change mute state based on frame type
+        should_mute = None
+
+        # Process frames to determine mute state
+        if isinstance(frame, StartFrame):
+            should_mute = await self._should_mute()
+        elif isinstance(frame, FunctionCallInProgressFrame):
             self._function_call_in_progress = True
-            await self._handle_mute_state(await self._should_mute())
+            should_mute = await self._should_mute()
         elif isinstance(frame, FunctionCallResultFrame):
             self._function_call_in_progress = False
-            await self._handle_mute_state(await self._should_mute())
-        # Handle bot speaking state changes
+            should_mute = await self._should_mute()
         elif isinstance(frame, BotStartedSpeakingFrame):
             self._bot_is_speaking = True
-            await self._handle_mute_state(await self._should_mute())
+            should_mute = await self._should_mute()
         elif isinstance(frame, BotStoppedSpeakingFrame):
             self._bot_is_speaking = False
-            await self._handle_mute_state(await self._should_mute())
+            if not self._first_speech_handled:
+                self._first_speech_handled = True
+            should_mute = await self._should_mute()
 
-        # Handle frame propagation
+        # Then push the original frame
         if isinstance(
             frame,
             (
@@ -147,13 +174,18 @@ class STTMuteFilter(FrameProcessor):
                 StopInterruptionFrame,
                 UserStartedSpeakingFrame,
                 UserStoppedSpeakingFrame,
+                InputAudioRawFrame,
             ),
         ):
             # Only pass VAD-related frames when not muted
             if not self.is_muted:
                 await self.push_frame(frame, direction)
             else:
-                logger.debug(f"{frame.__class__.__name__} suppressed - STT currently muted")
+                logger.trace(f"{frame.__class__.__name__} suppressed - STT currently muted")
         else:
             # Pass all other frames through
             await self.push_frame(frame, direction)
+
+        # Finally handle mute state change if needed
+        if should_mute is not None and should_mute != self.is_muted:
+            await self._handle_mute_state(should_mute)

src/pipecat/processors/frame_processor.py

@@ -5,7 +5,6 @@
 #
 
 import asyncio
-import inspect
 from enum import Enum
 from typing import Awaitable, Callable, Coroutine, Optional
 
@@ -23,8 +22,8 @@ from pipecat.frames.frames import (
 )
 from pipecat.metrics.metrics import LLMTokenUsage, MetricsData
 from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics
-from pipecat.utils.asyncio import TaskManager
-from pipecat.utils.utils import obj_count, obj_id
+from pipecat.utils.asyncio import BaseTaskManager
+from pipecat.utils.base_object import BaseObject
 
 
 class FrameDirection(Enum):
@@ -32,7 +31,7 @@ class FrameDirection(Enum):
     UPSTREAM = 2
 
 
-class FrameProcessor:
+class FrameProcessor(BaseObject):
     def __init__(
         self,
         *,
@@ -40,19 +39,16 @@ class FrameProcessor:
         metrics: Optional[FrameProcessorMetrics] = None,
         **kwargs,
     ):
-        self._id: int = obj_id()
-        self._name = name or f"{self.__class__.__name__}#{obj_count(self)}"
+        super().__init__(name=name)
         self._parent: Optional["FrameProcessor"] = None
         self._prev: Optional["FrameProcessor"] = None
         self._next: Optional["FrameProcessor"] = None
 
-        self._event_handlers: dict = {}
-
         # Clock
         self._clock: Optional[BaseClock] = None
 
         # Task Manager
-        self._task_manager: Optional[TaskManager] = None
+        self._task_manager: Optional[BaseTaskManager] = None
 
         # Other properties
         self._allow_interruptions = False
@@ -73,10 +69,11 @@ class FrameProcessor:
         self._metrics.set_processor_name(self.name)
 
         # Processors have an input queue. The input queue will be processed
-        # immediately (default) or it will block if `pause_processing_frames()` is
-        # called. To resume processing frames we need to call
-        # `resume_processing_frames()`.
+        # immediately (default) or it will block if `pause_processing_frames()`
+        # is called. To resume processing frames we need to call
+        # `resume_processing_frames()` which will wake up the event.
         self.__should_block_frames = False
+        self.__input_event = asyncio.Event()
         self.__input_frame_task: Optional[asyncio.Task] = None
 
         # Every processor in Pipecat should only output frames from a single
@@ -150,10 +147,13 @@ class FrameProcessor:
         await self.stop_ttfb_metrics()
         await self.stop_processing_metrics()
 
-    def create_task(self, coroutine: Coroutine) -> asyncio.Task:
+    def create_task(self, coroutine: Coroutine, name: Optional[str] = None) -> asyncio.Task:
         if not self._task_manager:
             raise Exception(f"{self} TaskManager is still not initialized.")
-        name = f"{self}::{coroutine.cr_code.co_name}"
+        if name:
+            name = f"{self}::{name}"
+        else:
+            name = f"{self}::{coroutine.cr_code.co_name}"
         return self._task_manager.create_task(coroutine, name)
 
     async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
@@ -167,6 +167,7 @@ class FrameProcessor:
         await self._task_manager.wait_for_task(task, timeout)
 
     async def cleanup(self):
+        await super().cleanup()
         await self.__cancel_input_task()
         await self.__cancel_push_task()
 
@@ -191,7 +192,7 @@ class FrameProcessor:
             raise Exception(f"{self} Clock is still not initialized.")
         return self._clock
 
-    def get_task_manager(self) -> TaskManager:
+    def get_task_manager(self) -> BaseTaskManager:
         if not self._task_manager:
             raise Exception(f"{self} TaskManager is still not initialized.")
         return self._task_manager
@@ -239,38 +240,29 @@ class FrameProcessor:
         elif isinstance(frame, StopInterruptionFrame):
             self._should_report_ttfb = True
         elif isinstance(frame, CancelFrame):
-            self._cancelling = True
+            await self.__cancel(frame)
 
     async def push_error(self, error: ErrorFrame):
         await self.push_frame(error, FrameDirection.UPSTREAM)
 
     async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
+        if not self._check_ready(frame):
+            return
+
         if isinstance(frame, SystemFrame):
             await self.__internal_push_frame(frame, direction)
         else:
             await self.__push_queue.put((frame, direction))
 
-    def event_handler(self, event_name: str):
-        def decorator(handler):
-            self.add_event_handler(event_name, handler)
-            return handler
-
-        return decorator
-
-    def add_event_handler(self, event_name: str, handler):
-        if event_name not in self._event_handlers:
-            raise Exception(f"Event handler {event_name} not registered")
-        self._event_handlers[event_name].append(handler)
-
-    def _register_event_handler(self, event_name: str):
-        if event_name in self._event_handlers:
-            raise Exception(f"Event handler {event_name} already registered")
-        self._event_handlers[event_name] = []
-
     async def __start(self, frame: StartFrame):
         self.__create_input_task()
         self.__create_push_task()
 
+    async def __cancel(self, frame: CancelFrame):
+        self._cancelling = True
+        await self.__cancel_input_task()
+        await self.__cancel_push_task()
+
     #
     # Handle interruptions
     #
@@ -319,11 +311,21 @@ class FrameProcessor:
             await self.push_error(ErrorFrame(str(e)))
             raise
 
+    def _check_ready(self, frame: Frame):
+        # If we are trying to push a frame but we still have no clock, it means
+        # we didn't process a StartFrame.
+        if not self._clock:
+            logger.error(
+                f"{self} not properly initialized, missing super().process_frame(frame, direction)?"
+            )
+            return False
+        return True
+
     def __create_input_task(self):
         if not self.__input_frame_task:
             self.__should_block_frames = False
+            self.__input_event.clear()
             self.__input_queue = asyncio.Queue()
-            self.__input_event = asyncio.Event()
             self.__input_frame_task = self.create_task(self.__input_frame_task_handler())
 
     async def __cancel_input_task(self):
@@ -366,16 +368,3 @@ class FrameProcessor:
             (frame, direction) = await self.__push_queue.get()
             await self.__internal_push_frame(frame, direction)
             self.__push_queue.task_done()
-
-    async def _call_event_handler(self, event_name: str, *args, **kwargs):
-        try:
-            for handler in self._event_handlers[event_name]:
-                if inspect.iscoroutinefunction(handler):
-                    await handler(self, *args, **kwargs)
-                else:
-                    handler(self, *args, **kwargs)
-        except Exception as e:
-            logger.exception(f"Exception in event handler {event_name}: {e}")
-
-    def __str__(self):
-        return self.name

src/pipecat/processors/frameworks/langchain.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-from typing import Union
+from typing import Optional, Union
 
 from loguru import logger
 
@@ -30,7 +30,7 @@ class LangchainProcessor(FrameProcessor):
         super().__init__()
         self._chain = chain
         self._transcript_key = transcript_key
-        self._participant_id: str | None = None
+        self._participant_id: Optional[str] = None
 
     def set_participant_id(self, participant_id: str):
         self._participant_id = participant_id

src/pipecat/processors/frameworks/rtvi.py

@@ -5,6 +5,7 @@
 #
 
 import asyncio
+import base64
 from dataclasses import dataclass
 from typing import (
     Any,
@@ -28,9 +29,11 @@ from pipecat.frames.frames import (
     CancelFrame,
     DataFrame,
     EndFrame,
+    EndTaskFrame,
     ErrorFrame,
     Frame,
     FunctionCallResultFrame,
+    InputAudioRawFrame,
     InterimTranscriptionFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
@@ -58,7 +61,9 @@ from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContextFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame
+from pipecat.transports.base_input import BaseInputTransport
+from pipecat.transports.base_output import BaseOutputTransport
+from pipecat.transports.base_transport import BaseTransport
 from pipecat.utils.string import match_endofsentence
 
 RTVI_PROTOCOL_VERSION = "0.3.0"
@@ -296,12 +301,6 @@ class RTVITextMessageData(BaseModel):
     text: str
 
 
-class RTVISearchResponseMessageData(BaseModel):
-    search_result: Optional[str]
-    rendered_content: Optional[str]
-    origins: List[LLMSearchOrigin]
-
-
 class RTVIBotTranscriptionMessage(BaseModel):
     label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
     type: Literal["bot-transcription"] = "bot-transcription"
@@ -314,12 +313,6 @@ class RTVIBotLLMTextMessage(BaseModel):
     data: RTVITextMessageData
 
 
-class RTVIBotLLMSearchResponseMessage(BaseModel):
-    label: Literal["rtvi-ai"] = "rtvi-ai"
-    type: Literal["bot-llm-search-response"] = "bot-llm-search-response"
-    data: RTVISearchResponseMessageData
-
-
 class RTVIBotTTSTextMessage(BaseModel):
     label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
     type: Literal["bot-tts-text"] = "bot-tts-text"
@@ -383,209 +376,35 @@ class RTVIMetricsMessage(BaseModel):
     data: Mapping[str, Any]
 
 
-class RTVIFrameProcessor(FrameProcessor):
-    def __init__(self, direction: FrameDirection = FrameDirection.DOWNSTREAM, **kwargs):
-        super().__init__(**kwargs)
-        self._direction = direction
-
-    async def _push_transport_message_urgent(self, model: BaseModel, exclude_none: bool = True):
-        frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none))
-        await self.push_frame(frame, self._direction)
+class RTVIServerMessage(BaseModel):
+    label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
+    type: Literal["server-message"] = "server-message"
+    data: Any
 
 
-class RTVISpeakingProcessor(RTVIFrameProcessor):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
+@dataclass
+class RTVIServerMessageFrame(SystemFrame):
+    """A frame for sending server messages to the client."""
 
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
+    data: Any
 
-        await self.push_frame(frame, direction)
-
-        if isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)):
-            await self._handle_interruptions(frame)
-        elif isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame)):
-            await self._handle_bot_speaking(frame)
-
-    async def _handle_interruptions(self, frame: Frame):
-        message = None
-        if isinstance(frame, UserStartedSpeakingFrame):
-            message = RTVIUserStartedSpeakingMessage()
-        elif isinstance(frame, UserStoppedSpeakingFrame):
-            message = RTVIUserStoppedSpeakingMessage()
-
-        if message:
-            await self._push_transport_message_urgent(message)
-
-    async def _handle_bot_speaking(self, frame: Frame):
-        message = None
-        if isinstance(frame, BotStartedSpeakingFrame):
-            message = RTVIBotStartedSpeakingMessage()
-        elif isinstance(frame, BotStoppedSpeakingFrame):
-            message = RTVIBotStoppedSpeakingMessage()
-
-        if message:
-            await self._push_transport_message_urgent(message)
-
-
-class RTVIUserTranscriptionProcessor(RTVIFrameProcessor):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        await self.push_frame(frame, direction)
-
-        if isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame)):
-            await self._handle_user_transcriptions(frame)
-
-    async def _handle_user_transcriptions(self, frame: Frame):
-        message = None
-        if isinstance(frame, TranscriptionFrame):
-            message = RTVIUserTranscriptionMessage(
-                data=RTVIUserTranscriptionMessageData(
-                    text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=True
-                )
-            )
-        elif isinstance(frame, InterimTranscriptionFrame):
-            message = RTVIUserTranscriptionMessage(
-                data=RTVIUserTranscriptionMessageData(
-                    text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=False
-                )
-            )
-
-        if message:
-            await self._push_transport_message_urgent(message)
-
-
-class RTVIUserLLMTextProcessor(RTVIFrameProcessor):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        await self.push_frame(frame, direction)
-
-        if isinstance(frame, OpenAILLMContextFrame):
-            await self._handle_context(frame)
-
-    async def _handle_context(self, frame: OpenAILLMContextFrame):
-        messages = frame.context.messages
-        if len(messages) > 0:
-            message = messages[-1]
-            if message["role"] == "user":
-                content = message["content"]
-                if isinstance(content, list):
-                    text = " ".join(item["text"] for item in content if "text" in item)
-                else:
-                    text = content
-                rtvi_message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=text))
-                await self._push_transport_message_urgent(rtvi_message)
-
-
-class RTVIBotTranscriptionProcessor(RTVIFrameProcessor):
-    def __init__(self):
-        super().__init__()
-        self._aggregation = ""
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        await self.push_frame(frame, direction)
-
-        if isinstance(frame, UserStartedSpeakingFrame):
-            await self._push_aggregation()
-        elif isinstance(frame, LLMTextFrame):
-            self._aggregation += frame.text
-            if match_endofsentence(self._aggregation):
-                await self._push_aggregation()
-
-    async def _push_aggregation(self):
-        if len(self._aggregation) > 0:
-            message = RTVIBotTranscriptionMessage(data=RTVITextMessageData(text=self._aggregation))
-            await self._push_transport_message_urgent(message)
-            self._aggregation = ""
-
-
-class RTVIBotLLMProcessor(RTVIFrameProcessor):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        await self.push_frame(frame, direction)
-
-        if isinstance(frame, LLMFullResponseStartFrame):
-            await self._push_transport_message_urgent(RTVIBotLLMStartedMessage())
-        elif isinstance(frame, LLMFullResponseEndFrame):
-            await self._push_transport_message_urgent(RTVIBotLLMStoppedMessage())
-        elif isinstance(frame, LLMTextFrame):
-            message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text))
-            await self._push_transport_message_urgent(message)
-
-
-class RTVIBotTTSProcessor(RTVIFrameProcessor):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        await self.push_frame(frame, direction)
-
-        if isinstance(frame, TTSStartedFrame):
-            await self._push_transport_message_urgent(RTVIBotTTSStartedMessage())
-        elif isinstance(frame, TTSStoppedFrame):
-            await self._push_transport_message_urgent(RTVIBotTTSStoppedMessage())
-        elif isinstance(frame, TTSTextFrame):
-            message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text))
-            await self._push_transport_message_urgent(message)
-
-
-class RTVIMetricsProcessor(RTVIFrameProcessor):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        await self.push_frame(frame, direction)
-
-        if isinstance(frame, MetricsFrame):
-            await self._handle_metrics(frame)
-
-    async def _handle_metrics(self, frame: MetricsFrame):
-        metrics = {}
-        for d in frame.data:
-            if isinstance(d, TTFBMetricsData):
-                if "ttfb" not in metrics:
-                    metrics["ttfb"] = []
-                metrics["ttfb"].append(d.model_dump(exclude_none=True))
-            elif isinstance(d, ProcessingMetricsData):
-                if "processing" not in metrics:
-                    metrics["processing"] = []
-                metrics["processing"].append(d.model_dump(exclude_none=True))
-            elif isinstance(d, LLMUsageMetricsData):
-                if "tokens" not in metrics:
-                    metrics["tokens"] = []
-                metrics["tokens"].append(d.value.model_dump(exclude_none=True))
-            elif isinstance(d, TTSUsageMetricsData):
-                if "characters" not in metrics:
-                    metrics["characters"] = []
-                metrics["characters"].append(d.model_dump(exclude_none=True))
-
-        message = RTVIMetricsMessage(data=metrics)
-        await self._push_transport_message_urgent(message)
+    def __str__(self):
+        return f"{self.name}(data: {self.data})"
 
 
 class RTVIObserver(BaseObserver):
-    """This is a pipeline frame observer that is used to send RTVI server
-    messages to clients. The observer does not handle incoming RTVI client
-    messages, which is done by the RTVIProcessor.
+    """Pipeline frame observer for RTVI server message handling.
 
+    This observer monitors pipeline frames and converts them into appropriate RTVI messages
+    for client communication. It handles various frame types including speech events,
+    transcriptions, LLM responses, and TTS events.
+
+    Note:
+        This observer only handles outgoing messages. Incoming RTVI client messages
+        are handled by the RTVIProcessor.
+
+    Args:
+        rtvi (FrameProcessor): The RTVI processor to push frames to.
     """
 
     def __init__(self, rtvi: FrameProcessor):
@@ -602,14 +421,28 @@ class RTVIObserver(BaseObserver):
         direction: FrameDirection,
         timestamp: int,
     ):
+        """Process a frame being pushed through the pipeline.
+
+        Args:
+            src: Source processor pushing the frame
+            dst: Destination processor receiving the frame
+            frame: The frame being pushed
+            direction: Direction of frame flow in pipeline
+            timestamp: Time when frame was pushed
+        """
         # If we have already seen this frame, let's skip it.
         if frame.id in self._frames_seen:
             return
-        self._frames_seen.add(frame.id)
+
+        # This tells whether the frame is already processed. If false, we will try
+        # again the next time we see the frame.
+        mark_as_seen = True
 
         if isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)):
             await self._handle_interruptions(frame)
-        elif isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame)):
+        elif isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame)) and (
+            direction == FrameDirection.UPSTREAM
+        ):
             await self._handle_bot_speaking(frame)
         elif isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame)):
             await self._handle_user_transcriptions(frame)
@@ -618,24 +451,37 @@ class RTVIObserver(BaseObserver):
         elif isinstance(frame, UserStartedSpeakingFrame):
             await self._push_bot_transcription()
         elif isinstance(frame, LLMFullResponseStartFrame):
-            await self._push_transport_message_urgent(RTVIBotLLMStartedMessage())
+            await self.push_transport_message_urgent(RTVIBotLLMStartedMessage())
         elif isinstance(frame, LLMFullResponseEndFrame):
-            await self._push_transport_message_urgent(RTVIBotLLMStoppedMessage())
+            await self.push_transport_message_urgent(RTVIBotLLMStoppedMessage())
         elif isinstance(frame, LLMTextFrame):
             await self._handle_llm_text_frame(frame)
-        elif isinstance(frame, LLMSearchResponseFrame):
-            await self._handle_llm_search_response_frame(frame)
         elif isinstance(frame, TTSStartedFrame):
-            await self._push_transport_message_urgent(RTVIBotTTSStartedMessage())
+            await self.push_transport_message_urgent(RTVIBotTTSStartedMessage())
         elif isinstance(frame, TTSStoppedFrame):
-            await self._push_transport_message_urgent(RTVIBotTTSStoppedMessage())
+            await self.push_transport_message_urgent(RTVIBotTTSStoppedMessage())
         elif isinstance(frame, TTSTextFrame):
-            message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text))
-            await self._push_transport_message_urgent(message)
+            if isinstance(src, BaseOutputTransport):
+                message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text))
+                await self.push_transport_message_urgent(message)
+            else:
+                mark_as_seen = False
         elif isinstance(frame, MetricsFrame):
             await self._handle_metrics(frame)
+        elif isinstance(frame, RTVIServerMessageFrame):
+            message = RTVIServerMessage(data=frame.data)
+            await self.push_transport_message_urgent(message)
 
-    async def _push_transport_message_urgent(self, model: BaseModel, exclude_none: bool = True):
+        if mark_as_seen:
+            self._frames_seen.add(frame.id)
+
+    async def push_transport_message_urgent(self, model: BaseModel, exclude_none: bool = True):
+        """Push an urgent transport message to the RTVI processor.
+
+        Args:
+            model: The message model to send
+            exclude_none: Whether to exclude None values from the model dump
+        """
         frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none))
         await self._rtvi.push_frame(frame)
 
@@ -644,7 +490,7 @@ class RTVIObserver(BaseObserver):
             message = RTVIBotTranscriptionMessage(
                 data=RTVITextMessageData(text=self._bot_transcription)
             )
-            await self._push_transport_message_urgent(message)
+            await self.push_transport_message_urgent(message)
             self._bot_transcription = ""
 
     async def _handle_interruptions(self, frame: Frame):
@@ -655,7 +501,7 @@ class RTVIObserver(BaseObserver):
             message = RTVIUserStoppedSpeakingMessage()
 
         if message:
-            await self._push_transport_message_urgent(message)
+            await self.push_transport_message_urgent(message)
 
     async def _handle_bot_speaking(self, frame: Frame):
         message = None
@@ -665,26 +511,16 @@ class RTVIObserver(BaseObserver):
             message = RTVIBotStoppedSpeakingMessage()
 
         if message:
-            await self._push_transport_message_urgent(message)
+            await self.push_transport_message_urgent(message)
 
     async def _handle_llm_text_frame(self, frame: LLMTextFrame):
         message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text))
-        await self._push_transport_message_urgent(message)
+        await self.push_transport_message_urgent(message)
 
         self._bot_transcription += frame.text
         if match_endofsentence(self._bot_transcription):
             await self._push_bot_transcription()
 
-    async def _handle_llm_search_response_frame(self, frame: LLMSearchResponseFrame):
-        message = RTVIBotLLMSearchResponseMessage(
-            data=RTVISearchResponseMessageData(
-                search_result=frame.search_result,
-                origins=frame.origins,
-                rendered_content=frame.rendered_content,
-            )
-        )
-        await self._push_transport_message_urgent(message)
-
     async def _handle_user_transcriptions(self, frame: Frame):
         message = None
         if isinstance(frame, TranscriptionFrame):
@@ -701,13 +537,26 @@ class RTVIObserver(BaseObserver):
             )
 
         if message:
-            await self._push_transport_message_urgent(message)
+            await self.push_transport_message_urgent(message)
 
     async def _handle_context(self, frame: OpenAILLMContextFrame):
+        """Process LLM context frames to extract user messages for the RTVI client."""
         try:
             messages = frame.context.messages
-            if len(messages) > 0:
-                message = messages[-1]
+            if not messages:
+                return
+
+            message = messages[-1]
+
+            # Handle Google LLM format (protobuf objects with attributes)
+            if hasattr(message, "role") and message.role == "user" and hasattr(message, "parts"):
+                text = "".join(part.text for part in message.parts if hasattr(part, "text"))
+                if text:
+                    rtvi_message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=text))
+                    await self.push_transport_message_urgent(rtvi_message)
+
+            # Handle OpenAI format (original implementation)
+            elif isinstance(message, dict):
                 if message["role"] == "user":
                     content = message["content"]
                     if isinstance(content, list):
@@ -715,8 +564,9 @@ class RTVIObserver(BaseObserver):
                     else:
                         text = content
                     rtvi_message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=text))
-                    await self._push_transport_message_urgent(rtvi_message)
-        except TypeError as e:
+                    await self.push_transport_message_urgent(rtvi_message)
+
+        except Exception as e:
             logger.warning(f"Caught an error while trying to handle context: {e}")
 
     async def _handle_metrics(self, frame: MetricsFrame):
@@ -740,7 +590,7 @@ class RTVIObserver(BaseObserver):
                 metrics["characters"].append(d.model_dump(exclude_none=True))
 
         message = RTVIMetricsMessage(data=metrics)
-        await self._push_transport_message_urgent(message)
+        await self.push_transport_message_urgent(message)
 
 
 class RTVIProcessor(FrameProcessor):
@@ -748,12 +598,13 @@ class RTVIProcessor(FrameProcessor):
         self,
         *,
         config: RTVIConfig = RTVIConfig(config=[]),
+        transport: Optional[BaseTransport] = None,
         **kwargs,
     ):
         super().__init__(**kwargs)
         self._config = config
 
-        self._pipeline: FrameProcessor | None = None
+        self._pipeline: Optional[FrameProcessor] = None
 
         self._bot_ready = False
         self._client_ready = False
@@ -773,8 +624,13 @@ class RTVIProcessor(FrameProcessor):
         self._register_event_handler("on_bot_started")
         self._register_event_handler("on_client_ready")
 
-    def observer(self) -> RTVIObserver:
-        return RTVIObserver(self)
+        self._input_transport = None
+        self._transport = transport
+        if self._transport:
+            input_transport = self._transport.input()
+            if isinstance(input_transport, BaseInputTransport):
+                self._input_transport = input_transport
+                self._input_transport.enable_audio_in_stream_on_start(False)
 
     def register_action(self, action: RTVIAction):
         id = self._action_id(action.service, action.action)
@@ -863,8 +719,10 @@ class RTVIProcessor(FrameProcessor):
             await self._pipeline.cleanup()
 
     async def _start(self, frame: StartFrame):
-        self._action_task = self.create_task(self._action_task_handler())
-        self._message_task = self.create_task(self._message_task_handler())
+        if not self._action_task:
+            self._action_task = self.create_task(self._action_task_handler())
+        if not self._message_task:
+            self._message_task = self.create_task(self._message_task_handler())
         await self._call_event_handler("on_bot_started")
 
     async def _stop(self, frame: EndFrame):
@@ -925,7 +783,7 @@ class RTVIProcessor(FrameProcessor):
                     update_config = RTVIUpdateConfig.model_validate(message.data)
                     await self._handle_update_config(message.id, update_config)
                 case "disconnect-bot":
-                    await self.push_frame(EndFrame())
+                    await self.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
                 case "action":
                     action = RTVIActionRun.model_validate(message.data)
                     action_frame = RTVIActionFrame(message_id=message.id, rtvi_action_run=action)
@@ -933,6 +791,8 @@ class RTVIProcessor(FrameProcessor):
                 case "llm-function-call-result":
                     data = RTVILLMFunctionCallResultData.model_validate(message.data)
                     await self._handle_function_call_result(data)
+                case "raw-audio" | "raw-audio-batch":
+                    await self._handle_audio_buffer(message.data)
 
                 case _:
                     await self._send_error_response(message.id, f"Unsupported type {message.type}")
@@ -945,9 +805,34 @@ class RTVIProcessor(FrameProcessor):
             logger.warning(f"Exception processing message: {e}")
 
     async def _handle_client_ready(self, request_id: str):
+        logger.debug("Received client-ready")
+        if self._input_transport:
+            self._input_transport.start_audio_in_streaming()
+
         self._client_ready_id = request_id
         await self.set_client_ready()
 
+    async def _handle_audio_buffer(self, data):
+        if not self._input_transport:
+            return
+
+        # Extract audio batch ensuring it's a list
+        audio_list = data.get("base64AudioBatch") or [data.get("base64Audio")]
+
+        try:
+            for base64_audio in filter(None, audio_list):  # Filter out None values
+                pcm_bytes = base64.b64decode(base64_audio)
+                frame = InputAudioRawFrame(
+                    audio=pcm_bytes,
+                    sample_rate=data["sampleRate"],
+                    num_channels=data["numChannels"],
+                )
+                await self._input_transport.push_audio_frame(frame)
+
+        except (KeyError, TypeError, ValueError) as e:
+            # Handle missing keys, decoding errors, and invalid types
+            logger.error(f"Error processing audio buffer: {e}")
+
     async def _handle_describe_config(self, request_id: str):
         services = list(self._registered_services.values())
         message = RTVIDescribeConfig(id=request_id, data=RTVIDescribeConfigData(config=services))
@@ -999,7 +884,7 @@ class RTVIProcessor(FrameProcessor):
         )
         await self.push_frame(frame)
 
-    async def _handle_action(self, request_id: str | None, data: RTVIActionRun):
+    async def _handle_action(self, request_id: Optional[str], data: RTVIActionRun):
         action_id = self._action_id(data.service, data.action)
         if action_id not in self._registered_actions:
             await self._send_error_response(request_id, f"Action {action_id} not registered")

src/pipecat/processors/gstreamer/pipeline_source.py

@@ -5,6 +5,7 @@
 #
 
 import asyncio
+from typing import Optional
 
 from loguru import logger
 from pydantic import BaseModel
@@ -38,7 +39,7 @@ class GStreamerPipelineSource(FrameProcessor):
     class OutputParams(BaseModel):
         video_width: int = 1280
         video_height: int = 720
-        audio_sample_rate: int = 24000
+        audio_sample_rate: Optional[int] = None
         audio_channels: int = 1
         clock_sync: bool = True
 
@@ -46,6 +47,7 @@ class GStreamerPipelineSource(FrameProcessor):
         super().__init__(**kwargs)
 
         self._out_params = out_params
+        self._sample_rate = 0
 
         Gst.init()
 
@@ -90,6 +92,7 @@ class GStreamerPipelineSource(FrameProcessor):
             await self.push_frame(frame, direction)
 
     async def _start(self, frame: StartFrame):
+        self._sample_rate = self._out_params.audio_sample_rate or frame.audio_out_sample_rate
         self._player.set_state(Gst.State.PLAYING)
 
     async def _stop(self, frame: EndFrame):
@@ -122,7 +125,7 @@ class GStreamerPipelineSource(FrameProcessor):
         audioresample = Gst.ElementFactory.make("audioresample", None)
         audiocapsfilter = Gst.ElementFactory.make("capsfilter", None)
         audiocaps = Gst.Caps.from_string(
-            f"audio/x-raw,format=S16LE,rate={self._out_params.audio_sample_rate},channels={self._out_params.audio_channels},layout=interleaved"
+            f"audio/x-raw,format=S16LE,rate={self._sample_rate},channels={self._out_params.audio_channels},layout=interleaved"
         )
         audiocapsfilter.set_property("caps", audiocaps)
         appsink_audio = Gst.ElementFactory.make("appsink", None)
@@ -188,7 +191,7 @@ class GStreamerPipelineSource(FrameProcessor):
         (_, info) = buffer.map(Gst.MapFlags.READ)
         frame = OutputAudioRawFrame(
             audio=info.data,
-            sample_rate=self._out_params.audio_sample_rate,
+            sample_rate=self._sample_rate,
             num_channels=self._out_params.audio_channels,
         )
         asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())

src/pipecat/processors/idle_frame_processor.py

@@ -30,6 +30,7 @@ class IdleFrameProcessor(FrameProcessor):
         self._callback = callback
         self._timeout = timeout
         self._types = types
+        self._idle_task = None
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
@@ -49,11 +50,13 @@ class IdleFrameProcessor(FrameProcessor):
                     self._idle_event.set()
 
     async def cleanup(self):
-        await self.cancel_task(self._idle_task)
+        if self._idle_task:
+            await self.cancel_task(self._idle_task)
 
     def _create_idle_task(self):
-        self._idle_event = asyncio.Event()
-        self._idle_task = self.create_task(self._idle_task_handler())
+        if not self._idle_task:
+            self._idle_event = asyncio.Event()
+            self._idle_task = self.create_task(self._idle_task_handler())
 
     async def _idle_task_handler(self):
         while True:

src/pipecat/processors/metrics/sentry.py

@@ -4,19 +4,14 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-import time
-
 from loguru import logger
 
 try:
     import sentry_sdk
-
-    sentry_available = sentry_sdk.is_initialized()
-    if not sentry_available:
-        logger.warning("Sentry SDK not initialized. Sentry features will be disabled.")
-except ImportError:
-    sentry_available = False
-    logger.warning("Sentry SDK not installed. Sentry features will be disabled.")
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error("In order to use Sentry, you need to `pip install pipecat-ai[sentry]`.")
+    raise Exception(f"Missing module: {e}")
 
 from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics
 
@@ -24,41 +19,44 @@ from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMet
 class SentryMetrics(FrameProcessorMetrics):
     def __init__(self):
         super().__init__()
-        self._ttfb_metrics_span = None
-        self._processing_metrics_span = None
+        self._ttfb_metrics_tx = None
+        self._processing_metrics_tx = None
+        self._sentry_available = sentry_sdk.is_initialized()
+        if not self._sentry_available:
+            logger.warning("Sentry SDK not initialized. Sentry features will be disabled.")
 
     async def start_ttfb_metrics(self, report_only_initial_ttfb):
-        if self._should_report_ttfb:
-            self._start_ttfb_time = time.time()
-            if sentry_available:
-                self._ttfb_metrics_span = sentry_sdk.start_span(
-                    op="ttfb",
-                    description=f"TTFB for {self._processor_name()}",
-                    start_timestamp=self._start_ttfb_time,
-                )
-                logger.debug(
-                    f"Sentry Span ID: {self._ttfb_metrics_span.span_id} Description: {self._ttfb_metrics_span.description} started."
-                )
-            self._should_report_ttfb = not report_only_initial_ttfb
+        await super().start_ttfb_metrics(report_only_initial_ttfb)
 
-    async def stop_ttfb_metrics(self):
-        stop_time = time.time()
-        if sentry_available:
-            self._ttfb_metrics_span.finish(end_timestamp=stop_time)
-
-    async def start_processing_metrics(self):
-        self._start_processing_time = time.time()
-        if sentry_available:
-            self._processing_metrics_span = sentry_sdk.start_span(
-                op="processing",
-                description=f"Processing for {self._processor_name()}",
-                start_timestamp=self._start_processing_time,
+        if self._should_report_ttfb and self._sentry_available:
+            self._ttfb_metrics_tx = sentry_sdk.start_transaction(
+                op="ttfb",
+                name=f"TTFB for {self._processor_name()}",
             )
             logger.debug(
-                f"Sentry Span ID: {self._processing_metrics_span.span_id} Description: {self._processing_metrics_span.description} started."
+                f"Sentry transaction started (ID: {self._ttfb_metrics_tx.span_id} Name: {self._ttfb_metrics_tx.name})"
+            )
+
+    async def stop_ttfb_metrics(self):
+        await super().stop_ttfb_metrics()
+
+        if self._sentry_available and self._ttfb_metrics_tx:
+            self._ttfb_metrics_tx.finish()
+
+    async def start_processing_metrics(self):
+        await super().start_processing_metrics()
+
+        if self._sentry_available:
+            self._processing_metrics_tx = sentry_sdk.start_transaction(
+                op="processing",
+                name=f"Processing for {self._processor_name()}",
+            )
+            logger.debug(
+                f"Sentry transaction started (ID: {self._processing_metrics_tx.span_id} Name: {self._processing_metrics_tx.name})"
             )
 
     async def stop_processing_metrics(self):
-        stop_time = time.time()
-        if sentry_available:
-            self._processing_metrics_span.finish(end_timestamp=stop_time)
+        await super().stop_processing_metrics()
+
+        if self._sentry_available and self._processing_metrics_tx:
+            self._processing_metrics_tx.finish()

src/pipecat/processors/transcript_processor.py

@@ -87,14 +87,65 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
         """Initialize processor with aggregation state."""
         super().__init__(**kwargs)
         self._current_text_parts: List[str] = []
-        self._aggregation_start_time: Optional[str] | None = None
+        self._aggregation_start_time: Optional[str] = None
 
     async def _emit_aggregated_text(self):
-        """Emit aggregated text as a transcript message."""
+        """Aggregates and emits text fragments as a transcript message.
+
+        This method uses a heuristic to automatically detect whether text fragments
+        use pre-spacing (spaces at the beginning of fragments) or not, and applies
+        the appropriate joining strategy. It handles fragments from different TTS
+        services with different formatting patterns.
+
+        Examples:
+            Pre-spaced fragments (concatenated):
+                ```
+                TTSTextFrame: ["Hello"]
+                TTSTextFrame: [" there"]
+                TTSTextFrame: ["!"]
+                TTSTextFrame: [" How"]
+                TTSTextFrame: ["'s"]
+                TTSTextFrame: [" it"]
+                TTSTextFrame: [" going"]
+                TTSTextFrame: ["?"]
+                ```
+                Result: "Hello there! How's it going?"
+
+            Word-by-word fragments (joined with spaces):
+                ```
+                TTSTextFrame: ["Hello"]
+                TTSTextFrame: ["there!"]
+                TTSTextFrame: ["How"]
+                TTSTextFrame: ["is"]
+                TTSTextFrame: ["it"]
+                TTSTextFrame: ["going?"]
+                ```
+                Result: "Hello there! How is it going?"
+        """
         if self._current_text_parts and self._aggregation_start_time:
-            content = " ".join(self._current_text_parts).strip()
+            # Heuristic to detect pre-spaced fragments
+            uses_prespacing = False
+            if len(self._current_text_parts) > 1:
+                # Check if any fragment after the first one starts with whitespace
+                has_spaced_parts = any(
+                    part and part[0].isspace() for part in self._current_text_parts[1:]
+                )
+                if has_spaced_parts:
+                    uses_prespacing = True
+
+            # Apply appropriate joining method
+            if uses_prespacing:
+                # Pre-spaced fragments - just concatenate
+                content = "".join(self._current_text_parts)
+            else:
+                # Word-by-word fragments - join with spaces
+                content = " ".join(self._current_text_parts)
+
+            # Clean up any excessive whitespace
+            content = content.strip()
+
             if content:
-                logger.debug(f"Emitting aggregated assistant message: {content}")
+                logger.trace(f"Emitting aggregated assistant message: {content}")
                 message = TranscriptionMessage(
                     role="assistant",
                     content=content,
@@ -102,7 +153,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
                 )
                 await self._emit_update([message])
             else:
-                logger.debug("No content to emit after stripping whitespace")
+                logger.trace("No content to emit after stripping whitespace")
 
             # Reset aggregation state
             self._current_text_parts = []

src/pipecat/processors/user_idle_processor.py

@@ -102,7 +102,7 @@ class UserIdleProcessor(FrameProcessor):
 
     def _create_idle_task(self) -> None:
         """Creates the idle task if it hasn't been created yet."""
-        if self._idle_task is None:
+        if not self._idle_task:
             self._idle_task = self.create_task(self._idle_task_handler())
 
     @property
@@ -112,7 +112,7 @@ class UserIdleProcessor(FrameProcessor):
 
     async def _stop(self) -> None:
         """Stops and cleans up the idle monitoring task."""
-        if self._idle_task is not None:
+        if self._idle_task:
             await self.cancel_task(self._idle_task)
             self._idle_task = None
 

src/pipecat/serializers/base_serializer.py

@@ -7,7 +7,7 @@
 from abc import ABC, abstractmethod
 from enum import Enum
 
-from pipecat.frames.frames import Frame
+from pipecat.frames.frames import Frame, StartFrame
 
 
 class FrameSerializerType(Enum):
@@ -21,6 +21,9 @@ class FrameSerializer(ABC):
     def type(self) -> FrameSerializerType:
         pass
 
+    async def setup(self, frame: StartFrame):
+        pass
+
     @abstractmethod
     async def serialize(self, frame: Frame) -> str | bytes | None:
         pass

src/pipecat/serializers/telnyx.py

@@ -6,6 +6,7 @@
 
 import base64
 import json
+from typing import Optional
 
 from pydantic import BaseModel
 
@@ -22,6 +23,7 @@ from pipecat.frames.frames import (
     InputAudioRawFrame,
     InputDTMFFrame,
     KeypadEntry,
+    StartFrame,
     StartInterruptionFrame,
 )
 from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
@@ -29,8 +31,8 @@ from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializer
 
 class TelnyxFrameSerializer(FrameSerializer):
     class InputParams(BaseModel):
-        telnyx_sample_rate: int = 8000
-        sample_rate: int = 16000
+        telnyx_sample_rate: int = 8000  # Default Telnyx rate (8kHz)
+        sample_rate: Optional[int] = None  # Pipeline input rate
         inbound_encoding: str = "PCMU"
         outbound_encoding: str = "PCMU"
 
@@ -46,23 +48,30 @@ class TelnyxFrameSerializer(FrameSerializer):
         params.inbound_encoding = inbound_encoding
         self._params = params
 
+        self._telnyx_sample_rate = self._params.telnyx_sample_rate
+        self._sample_rate = 0  # Pipeline input rate
+
         self._resampler = create_default_resampler()
 
     @property
     def type(self) -> FrameSerializerType:
         return FrameSerializerType.TEXT
 
+    async def setup(self, frame: StartFrame):
+        self._sample_rate = self._params.sample_rate or frame.audio_in_sample_rate
+
     async def serialize(self, frame: Frame) -> str | bytes | None:
         if isinstance(frame, AudioRawFrame):
             data = frame.audio
 
+            # Output: Convert PCM at frame's rate to 8kHz encoded for Telnyx
             if self._params.inbound_encoding == "PCMU":
                 serialized_data = await pcm_to_ulaw(
-                    data, frame.sample_rate, self._params.telnyx_sample_rate, self._resampler
+                    data, frame.sample_rate, self._telnyx_sample_rate, self._resampler
                 )
             elif self._params.inbound_encoding == "PCMA":
                 serialized_data = await pcm_to_alaw(
-                    data, frame.sample_rate, self._params.telnyx_sample_rate, self._resampler
+                    data, frame.sample_rate, self._telnyx_sample_rate, self._resampler
                 )
             else:
                 raise ValueError(f"Unsupported encoding: {self._params.inbound_encoding}")
@@ -86,25 +95,26 @@ class TelnyxFrameSerializer(FrameSerializer):
             payload_base64 = message["media"]["payload"]
             payload = base64.b64decode(payload_base64)
 
+            # Input: Convert Telnyx's 8kHz encoded audio to PCM at pipeline input rate
             if self._params.outbound_encoding == "PCMU":
                 deserialized_data = await ulaw_to_pcm(
                     payload,
-                    self._params.telnyx_sample_rate,
-                    self._params.sample_rate,
+                    self._telnyx_sample_rate,
+                    self._sample_rate,
                     self._resampler,
                 )
             elif self._params.outbound_encoding == "PCMA":
                 deserialized_data = await alaw_to_pcm(
                     payload,
-                    self._params.telnyx_sample_rate,
-                    self._params.sample_rate,
+                    self._telnyx_sample_rate,
+                    self._sample_rate,
                     self._resampler,
                 )
             else:
                 raise ValueError(f"Unsupported encoding: {self._params.outbound_encoding}")
 
             audio_frame = InputAudioRawFrame(
-                audio=deserialized_data, num_channels=1, sample_rate=self._params.sample_rate
+                audio=deserialized_data, num_channels=1, sample_rate=self._sample_rate
             )
             return audio_frame
         elif message["event"] == "dtmf":

src/pipecat/serializers/twilio.py

@@ -6,6 +6,7 @@
 
 import base64
 import json
+from typing import Optional
 
 from pydantic import BaseModel
 
@@ -16,6 +17,7 @@ from pipecat.frames.frames import (
     InputAudioRawFrame,
     InputDTMFFrame,
     KeypadEntry,
+    StartFrame,
     StartInterruptionFrame,
     TransportMessageFrame,
     TransportMessageUrgentFrame,
@@ -25,19 +27,25 @@ from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializer
 
 class TwilioFrameSerializer(FrameSerializer):
     class InputParams(BaseModel):
-        twilio_sample_rate: int = 8000
-        sample_rate: int = 16000
+        twilio_sample_rate: int = 8000  # Default Twilio rate (8kHz)
+        sample_rate: Optional[int] = None  # Pipeline input rate
 
     def __init__(self, stream_sid: str, params: InputParams = InputParams()):
         self._stream_sid = stream_sid
         self._params = params
 
+        self._twilio_sample_rate = self._params.twilio_sample_rate
+        self._sample_rate = 0  # Pipeline input rate
+
         self._resampler = create_default_resampler()
 
     @property
     def type(self) -> FrameSerializerType:
         return FrameSerializerType.TEXT
 
+    async def setup(self, frame: StartFrame):
+        self._sample_rate = self._params.sample_rate or frame.audio_in_sample_rate
+
     async def serialize(self, frame: Frame) -> str | bytes | None:
         if isinstance(frame, StartInterruptionFrame):
             answer = {"event": "clear", "streamSid": self._stream_sid}
@@ -45,8 +53,9 @@ class TwilioFrameSerializer(FrameSerializer):
         elif isinstance(frame, AudioRawFrame):
             data = frame.audio
 
+            # Output: Convert PCM at frame's rate to 8kHz μ-law for Twilio
             serialized_data = await pcm_to_ulaw(
-                data, frame.sample_rate, self._params.twilio_sample_rate, self._resampler
+                data, frame.sample_rate, self._twilio_sample_rate, self._resampler
             )
             payload = base64.b64encode(serialized_data).decode("utf-8")
             answer = {
@@ -66,11 +75,12 @@ class TwilioFrameSerializer(FrameSerializer):
             payload_base64 = message["media"]["payload"]
             payload = base64.b64decode(payload_base64)
 
+            # Input: Convert Twilio's 8kHz μ-law to PCM at pipeline input rate
             deserialized_data = await ulaw_to_pcm(
-                payload, self._params.twilio_sample_rate, self._params.sample_rate, self._resampler
+                payload, self._twilio_sample_rate, self._sample_rate, self._resampler
             )
             audio_frame = InputAudioRawFrame(
-                audio=deserialized_data, num_channels=1, sample_rate=self._params.sample_rate
+                audio=deserialized_data, num_channels=1, sample_rate=self._sample_rate
             )
             return audio_frame
         elif message["event"] == "dtmf":

src/pipecat/services/ai_services.py

@@ -8,17 +8,24 @@ import asyncio
 import io
 import wave
 from abc import abstractmethod
-from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple
+from dataclasses import dataclass
+from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Set, Tuple, Type
 
 from loguru import logger
 
-from pipecat.audio.utils import calculate_audio_volume, exp_smoothing
+from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
+from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter
 from pipecat.frames.frames import (
     AudioRawFrame,
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
     CancelFrame,
     EndFrame,
     ErrorFrame,
     Frame,
+    FunctionCallCancelFrame,
+    FunctionCallInProgressFrame,
+    FunctionCallResultFrame,
     InterimTranscriptionFrame,
     LLMFullResponseEndFrame,
     StartFrame,
@@ -34,14 +41,18 @@ from pipecat.frames.frames import (
     TTSTextFrame,
     TTSUpdateSettingsFrame,
     UserImageRequestFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
     VisionImageRawFrame,
 )
 from pipecat.metrics.metrics import MetricsData
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.websocket_service import WebsocketService
 from pipecat.transcriptions.language import Language
-from pipecat.utils.string import match_endofsentence
+from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
 from pipecat.utils.text.base_text_filter import BaseTextFilter
+from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator
 from pipecat.utils.time import seconds_to_nanoseconds
 
 
@@ -75,13 +86,13 @@ class AIService(FrameProcessor):
         )
 
         for key, value in settings.items():
-            print("Update request for:", key, value)
+            logger.debug("Update request for:", key, value)
 
             if key in self._settings:
                 logger.info(f"Updating LLM setting {key} to: [{value}]")
                 self._settings[key] = value
             elif key in SessionProperties.model_fields:
-                print("Attempting to update", key, value)
+                logger.debug("Attempting to update", key, value)
 
                 try:
                     from pipecat.services.openai_realtime_beta.events import (
@@ -131,32 +142,90 @@ class AIService(FrameProcessor):
                     await self.push_frame(f)
 
 
+@dataclass
+class FunctionEntry:
+    function_name: Optional[str]
+    callback: Any  # TODO(aleix): add proper typing.
+    cancel_on_interruption: bool
+
+
 class LLMService(AIService):
     """This class is a no-op but serves as a base class for LLM services."""
 
+    # OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations.
+    # However, subclasses should override this with a more specific adapter when necessary.
+    adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self._callbacks = {}
+        self._functions = {}
         self._start_callbacks = {}
+        self._adapter = self.adapter_class()
+        self._function_call_tasks: Set[Tuple[asyncio.Task, str, str]] = set()
 
-    # TODO-CB: callback function type
-    def register_function(self, function_name: str | None, callback, start_callback=None):
+        self._register_event_handler("on_completion_timeout")
+
+    def get_llm_adapter(self) -> BaseLLMAdapter:
+        return self._adapter
+
+    def create_context_aggregator(
+        self,
+        context: OpenAILLMContext,
+        *,
+        user_kwargs: Mapping[str, Any] = {},
+        assistant_kwargs: Mapping[str, Any] = {},
+    ) -> Any:
+        pass
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, StartInterruptionFrame):
+            await self._handle_interruptions(frame)
+
+    async def _handle_interruptions(self, frame: StartInterruptionFrame):
+        for function_name, entry in self._functions.items():
+            if entry.cancel_on_interruption:
+                await self._cancel_function_call(function_name)
+
+    def register_function(
+        self,
+        function_name: Optional[str],
+        callback: Any,
+        start_callback=None,
+        *,
+        cancel_on_interruption: bool = False,
+    ):
         # Registering a function with the function_name set to None will run that callback
         # for all functions
-        self._callbacks[function_name] = callback
-        # QUESTION FOR CB: maybe this isn't needed anymore?
+        self._functions[function_name] = FunctionEntry(
+            function_name=function_name,
+            callback=callback,
+            cancel_on_interruption=cancel_on_interruption,
+        )
+
+        # Start callbacks are now deprecated.
         if start_callback:
+            import warnings
+
+            with warnings.catch_warnings():
+                warnings.simplefilter("always")
+                warnings.warn(
+                    "Parameter 'start_callback' is deprecated, just put your code on top of the actual function call instead.",
+                    DeprecationWarning,
+                )
+
             self._start_callbacks[function_name] = start_callback
 
-    def unregister_function(self, function_name: str | None):
-        del self._callbacks[function_name]
+    def unregister_function(self, function_name: Optional[str]):
+        del self._functions[function_name]
         if self._start_callbacks[function_name]:
             del self._start_callbacks[function_name]
 
     def has_function(self, function_name: str):
-        if None in self._callbacks.keys():
+        if None in self._functions.keys():
             return True
-        return function_name in self._callbacks.keys()
+        return function_name in self._functions.keys()
 
     async def call_function(
         self,
@@ -166,35 +235,144 @@ class LLMService(AIService):
         function_name: str,
         arguments: str,
         run_llm: bool = True,
-    ) -> None:
-        f = None
-        if function_name in self._callbacks.keys():
-            f = self._callbacks[function_name]
-        elif None in self._callbacks.keys():
-            f = self._callbacks[None]
-        else:
-            return None
-        await context.call_function(
-            f,
-            function_name=function_name,
-            tool_call_id=tool_call_id,
-            arguments=arguments,
-            llm=self,
-            run_llm=run_llm,
+    ):
+        if not function_name in self._functions.keys() and not None in self._functions.keys():
+            return
+
+        task = self.create_task(
+            self._run_function_call(context, tool_call_id, function_name, arguments, run_llm)
         )
 
-    # QUESTION FOR CB: maybe this isn't needed anymore?
+        self._function_call_tasks.add((task, tool_call_id, function_name))
+
+        task.add_done_callback(self._function_call_task_finished)
+
     async def call_start_function(self, context: OpenAILLMContext, function_name: str):
         if function_name in self._start_callbacks.keys():
             await self._start_callbacks[function_name](function_name, self, context)
         elif None in self._start_callbacks.keys():
             return await self._start_callbacks[None](function_name, self, context)
 
-    async def request_image_frame(self, user_id: str, *, text_content: str | None = None):
+    async def request_image_frame(
+        self,
+        user_id: str,
+        *,
+        function_name: Optional[str] = None,
+        tool_call_id: Optional[str] = None,
+        text_content: Optional[str] = None,
+    ):
         await self.push_frame(
-            UserImageRequestFrame(user_id=user_id, context=text_content), FrameDirection.UPSTREAM
+            UserImageRequestFrame(
+                user_id=user_id,
+                function_name=function_name,
+                tool_call_id=tool_call_id,
+                context=text_content,
+            ),
+            FrameDirection.UPSTREAM,
         )
 
+    async def _run_function_call(
+        self,
+        context: OpenAILLMContext,
+        tool_call_id: str,
+        function_name: str,
+        arguments: str,
+        run_llm: bool = True,
+    ):
+        if function_name in self._functions.keys():
+            entry = self._functions[function_name]
+        elif None in self._functions.keys():
+            entry = self._functions[None]
+        else:
+            return
+
+        logger.debug(
+            f"{self} Calling function [{function_name}:{tool_call_id}] with arguments {arguments}"
+        )
+
+        # NOTE(aleix): This needs to be removed after we remove the deprecation.
+        await self.call_start_function(context, function_name)
+
+        # Push a SystemFrame downstream. This frame will let our assistant context aggregator
+        # know that we are in the middle of a function call. Some contexts/aggregators may
+        # not need this. But some definitely do (Anthropic, for example).
+        # Also push a SystemFrame upstream for use by other processors, like STTMuteFilter.
+        progress_frame_downstream = FunctionCallInProgressFrame(
+            function_name=function_name,
+            tool_call_id=tool_call_id,
+            arguments=arguments,
+            cancel_on_interruption=entry.cancel_on_interruption,
+        )
+        progress_frame_upstream = FunctionCallInProgressFrame(
+            function_name=function_name,
+            tool_call_id=tool_call_id,
+            arguments=arguments,
+            cancel_on_interruption=entry.cancel_on_interruption,
+        )
+
+        # Push frame both downstream and upstream
+        await self.push_frame(progress_frame_downstream, FrameDirection.DOWNSTREAM)
+        await self.push_frame(progress_frame_upstream, FrameDirection.UPSTREAM)
+
+        # Define a callback function that pushes a FunctionCallResultFrame upstream & downstream.
+        async def function_call_result_callback(result, *, properties=None):
+            result_frame_downstream = FunctionCallResultFrame(
+                function_name=function_name,
+                tool_call_id=tool_call_id,
+                arguments=arguments,
+                result=result,
+                properties=properties,
+            )
+            result_frame_upstream = FunctionCallResultFrame(
+                function_name=function_name,
+                tool_call_id=tool_call_id,
+                arguments=arguments,
+                result=result,
+                properties=properties,
+            )
+
+            await self.push_frame(result_frame_downstream, FrameDirection.DOWNSTREAM)
+            await self.push_frame(result_frame_upstream, FrameDirection.UPSTREAM)
+
+        await entry.callback(
+            function_name, tool_call_id, arguments, self, context, function_call_result_callback
+        )
+
+    async def _cancel_function_call(self, function_name: str):
+        cancelled_tasks = set()
+        for task, tool_call_id, name in self._function_call_tasks:
+            if name == function_name:
+                # We remove the callback because we are going to cancel the task
+                # now, otherwise we will be removing it from the set while we
+                # are iterating.
+                task.remove_done_callback(self._function_call_task_finished)
+
+                logger.debug(f"{self} Cancelling function call [{name}:{tool_call_id}]...")
+
+                await self.cancel_task(task)
+
+                frame = FunctionCallCancelFrame(
+                    function_name=function_name, tool_call_id=tool_call_id
+                )
+                await self.push_frame(frame)
+
+                logger.debug(f"{self} Function call [{name}:{tool_call_id}] has been cancelled")
+
+                cancelled_tasks.add(task)
+
+        # Remove all cancelled tasks from our set.
+        for task in cancelled_tasks:
+            self._function_call_task_finished(task)
+
+    def _function_call_task_finished(self, task: asyncio.Task):
+        tuple_to_remove = next((t for t in self._function_call_tasks if t[0] == task), None)
+        if tuple_to_remove:
+            self._function_call_tasks.discard(tuple_to_remove)
+            # The task is finished so this should exit immediately. We need to
+            # do this because otherwise the task manager would report a dangling
+            # task if we don't remove it.
+            asyncio.run_coroutine_threadsafe(self.wait_for_task(task), self.get_event_loop())
+
 
 class TTSService(AIService):
     def __init__(
@@ -207,13 +385,19 @@ class TTSService(AIService):
         # if True, TTSService will push TTSStoppedFrames, otherwise subclass must do it
         push_stop_frames: bool = False,
         # if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame
-        stop_frame_timeout_s: float = 1.0,
+        stop_frame_timeout_s: float = 2.0,
         # if True, TTSService will push silence audio frames after TTSStoppedFrame
         push_silence_after_stop: bool = False,
         # if push_silence_after_stop is True, send this amount of audio silence
         silence_time_s: float = 2.0,
+        # if True, we will pause processing frames while we are receiving audio
+        pause_frame_processing: bool = False,
         # TTS output sample rate
-        sample_rate: int = 24000,
+        sample_rate: Optional[int] = None,
+        # Text aggregator to aggregate incoming tokens and decide when to push to the TTS.
+        text_aggregator: Optional[BaseTextAggregator] = None,
+        # Text filter executed after text has been aggregated.
+        text_filters: Sequence[BaseTextFilter] = [],
         text_filter: Optional[BaseTextFilter] = None,
         **kwargs,
     ):
@@ -224,15 +408,28 @@ class TTSService(AIService):
         self._stop_frame_timeout_s: float = stop_frame_timeout_s
         self._push_silence_after_stop: bool = push_silence_after_stop
         self._silence_time_s: float = silence_time_s
-        self._sample_rate: int = sample_rate
+        self._pause_frame_processing: bool = pause_frame_processing
+        self._init_sample_rate = sample_rate
+        self._sample_rate = 0
         self._voice_id: str = ""
         self._settings: Dict[str, Any] = {}
-        self._text_filter: Optional[BaseTextFilter] = text_filter
+        self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
+        self._text_filters: Sequence[BaseTextFilter] = text_filters
+        if text_filter:
+            import warnings
+
+            with warnings.catch_warnings():
+                warnings.simplefilter("always")
+                warnings.warn(
+                    "Parameter 'text_filter' is deprecated, use 'text_filters' instead.",
+                    DeprecationWarning,
+                )
+            self._text_filters = [text_filter]
 
         self._stop_frame_task: Optional[asyncio.Task] = None
         self._stop_frame_queue: asyncio.Queue = asyncio.Queue()
 
-        self._current_sentence: str = ""
+        self._processing_text: bool = False
 
     @property
     def sample_rate(self) -> int:
@@ -244,21 +441,24 @@ class TTSService(AIService):
     def set_voice(self, voice: str):
         self._voice_id = voice
 
-    @abstractmethod
-    async def flush_audio(self):
-        pass
-
-    def language_to_service_language(self, language: Language) -> str | None:
-        return Language(language)
-
     # Converts the text to audio.
     @abstractmethod
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
         pass
 
+    def language_to_service_language(self, language: Language) -> Optional[str]:
+        return Language(language)
+
+    async def update_setting(self, key: str, value: Any):
+        pass
+
+    async def flush_audio(self):
+        pass
+
     async def start(self, frame: StartFrame):
         await super().start(frame)
-        if self._push_stop_frames:
+        self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
+        if self._push_stop_frames and not self._stop_frame_task:
             self._stop_frame_task = self.create_task(self._stop_frame_handler())
 
     async def stop(self, frame: EndFrame):
@@ -284,8 +484,9 @@ class TTSService(AIService):
                 self.set_model_name(value)
             elif key == "voice":
                 self.set_voice(value)
-            elif key == "text_filter" and self._text_filter:
-                self._text_filter.update_settings(value)
+            elif key == "text_filter":
+                for filter in self._text_filters:
+                    filter.update_settings(value)
             else:
                 logger.warning(f"Unknown setting for TTS service: {key}")
 
@@ -294,6 +495,7 @@ class TTSService(AIService):
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
+
         if (
             isinstance(frame, TextFrame)
             and not isinstance(frame, InterimTranscriptionFrame)
@@ -302,9 +504,16 @@ class TTSService(AIService):
             await self._process_text_frame(frame)
         elif isinstance(frame, StartInterruptionFrame):
             await self._handle_interruption(frame, direction)
+            await self.push_frame(frame, direction)
         elif isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
-            sentence = self._current_sentence
-            self._current_sentence = ""
+            # We pause processing incoming frames if the LLM response included
+            # text (it might be that it's only a function calling response). We
+            # pause to avoid audio overlapping.
+            await self._maybe_pause_frame_processing()
+
+            sentence = self._text_aggregator.text
+            self._text_aggregator.reset()
+            self._processing_text = False
             await self._push_tts_frames(sentence)
             if isinstance(frame, LLMFullResponseEndFrame):
                 if self._push_text_frames:
@@ -312,10 +521,19 @@ class TTSService(AIService):
             else:
                 await self.push_frame(frame, direction)
         elif isinstance(frame, TTSSpeakFrame):
+            # Store if we were processing text or not so we can set it back.
+            processing_text = self._processing_text
             await self._push_tts_frames(frame.text)
+            # We pause processing incoming frames because we are sending data to
+            # the TTS. We pause to avoid audio overlapping.
+            await self._maybe_pause_frame_processing()
             await self.flush_audio()
+            self._processing_text = processing_text
         elif isinstance(frame, TTSUpdateSettingsFrame):
             await self._update_settings(frame.settings)
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            await self._maybe_resume_frame_processing()
+            await self.push_frame(frame, direction)
         else:
             await self.push_frame(frame, direction)
 
@@ -341,37 +559,54 @@ class TTSService(AIService):
             await self._stop_frame_queue.put(frame)
 
     async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
-        self._current_sentence = ""
-        if self._text_filter:
-            self._text_filter.handle_interruption()
-        await self.push_frame(frame, direction)
+        self._processing_text = False
+        self._text_aggregator.handle_interruption()
+        for filter in self._text_filters:
+            filter.handle_interruption()
+
+    async def _maybe_pause_frame_processing(self):
+        if self._processing_text and self._pause_frame_processing:
+            await self.pause_processing_frames()
+
+    async def _maybe_resume_frame_processing(self):
+        if self._pause_frame_processing:
+            await self.resume_processing_frames()
 
     async def _process_text_frame(self, frame: TextFrame):
-        text: str | None = None
+        text: Optional[str] = None
         if not self._aggregate_sentences:
             text = frame.text
         else:
-            self._current_sentence += frame.text
-            eos_end_marker = match_endofsentence(self._current_sentence)
-            if eos_end_marker:
-                text = self._current_sentence[:eos_end_marker]
-                self._current_sentence = self._current_sentence[eos_end_marker:]
+            text = self._text_aggregator.aggregate(frame.text)
 
         if text:
             await self._push_tts_frames(text)
 
     async def _push_tts_frames(self, text: str):
+        # Remove leading newlines only
+        text = text.lstrip("\n")
+
         # Don't send only whitespace. This causes problems for some TTS models. But also don't
         # strip all whitespace, as whitespace can influence prosody.
         if not text.strip():
             return
 
+        # This is just a flag that indicates if we sent something to the TTS
+        # service. It will be cleared if we sent text because of a TTSSpeakFrame
+        # or when we received an LLMFullResponseEndFrame
+        self._processing_text = True
+
         await self.start_processing_metrics()
-        if self._text_filter:
-            self._text_filter.reset_interruption()
-            text = self._text_filter.filter(text)
+
+        # Process all filter.
+        for filter in self._text_filters:
+            filter.reset_interruption()
+            text = filter.filter(text)
+
         await self.process_generator(self.run_tts(text))
+
         await self.stop_processing_metrics()
+
         if self._push_text_frames:
             # We send the original text after the audio. This way, if we are
             # interrupted, the text is not added to the assistant context.
@@ -395,6 +630,12 @@ class TTSService(AIService):
 
 
 class WordTTSService(TTSService):
+    """This is a base class for TTS services that support word timestamps. Word
+    timestamps are useful to synchronize audio with text of the spoken
+    words. This way only the spoken words are added to the conversation context.
+
+    """
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self._initial_word_timestamp = -1
@@ -414,7 +655,7 @@ class WordTTSService(TTSService):
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
-        await self._create_words_task()
+        self._create_words_task()
 
     async def stop(self, frame: EndFrame):
         await super().stop(frame)
@@ -434,8 +675,9 @@ class WordTTSService(TTSService):
         await super()._handle_interruption(frame, direction)
         self.reset_word_timestamps()
 
-    async def _create_words_task(self):
-        self._words_task = self.create_task(self._words_task_handler())
+    def _create_words_task(self):
+        if not self._words_task:
+            self._words_task = self.create_task(self._words_task_handler())
 
     async def _stop_words_task(self):
         if self._words_task:
@@ -464,12 +706,250 @@ class WordTTSService(TTSService):
             self._words_queue.task_done()
 
 
+class WebsocketTTSService(TTSService, WebsocketService):
+    """This is a base class for websocket-based TTS services.
+
+    If an error occurs with the websocket, an "on_connection_error" event will
+    be triggered:
+
+       @tts.event_handler("on_connection_error")
+       async def on_connection_error(tts: TTSService, error: str):
+           ...
+
+    """
+
+    def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
+        TTSService.__init__(self, **kwargs)
+        WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs)
+        self._register_event_handler("on_connection_error")
+
+    async def _report_error(self, error: ErrorFrame):
+        await self._call_event_handler("on_connection_error", error.error)
+        await self.push_error(error)
+
+
+class InterruptibleTTSService(WebsocketTTSService):
+    """This is a base class for websocket-based TTS services that don't support
+    word timestamps and that don't offer a way to correlate the generated audio
+    to the requested text.
+
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        # Indicates if the bot is speaking. If the bot is not speaking we don't
+        # need to reconnect when the user speaks. If the bot is speaking and the
+        # user interrupts we need to reconnect.
+        self._bot_speaking = False
+
+    async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
+        await super()._handle_interruption(frame, direction)
+        if self._bot_speaking:
+            await self._disconnect()
+            await self._connect()
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, BotStartedSpeakingFrame):
+            self._bot_speaking = True
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            self._bot_speaking = False
+
+
+class WebsocketWordTTSService(WordTTSService, WebsocketService):
+    """This is a base class for websocket-based TTS services that support word
+    timestamps.
+
+    If an error occurs with the websocket a "on_connection_error" event will be
+    triggered:
+
+       @tts.event_handler("on_connection_error")
+       async def on_connection_error(tts: TTSService, error: str):
+           ...
+
+    """
+
+    def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
+        WordTTSService.__init__(self, **kwargs)
+        WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs)
+        self._register_event_handler("on_connection_error")
+
+    async def _report_error(self, error: ErrorFrame):
+        await self._call_event_handler("on_connection_error", error.error)
+        await self.push_error(error)
+
+
+class InterruptibleWordTTSService(WebsocketWordTTSService):
+    """This is a base class for websocket-based TTS services that support word
+    timestamps but don't offer a way to correlate the generated audio to the
+    requested text.
+
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        # Indicates if the bot is speaking. If the bot is not speaking we don't
+        # need to reconnect when the user speaks. If the bot is speaking and the
+        # user interrupts we need to reconnect.
+        self._bot_speaking = False
+
+    async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
+        await super()._handle_interruption(frame, direction)
+        if self._bot_speaking:
+            await self._disconnect()
+            await self._connect()
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, BotStartedSpeakingFrame):
+            self._bot_speaking = True
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            self._bot_speaking = False
+
+
+class AudioContextWordTTSService(WebsocketWordTTSService):
+    """This is a base class for websocket-based TTS services that support word
+    timestamps and also allow correlating the generated audio with the requested
+    text.
+
+    Each request could be multiple sentences long which are grouped by
+    context. For this to work, the TTS service needs to support handling
+    multiple requests at once (i.e. multiple simultaneous contexts).
+
+    The audio received from the TTS will be played in context order. That is, if
+    we requested audio for a context "A" and then audio for context "B", the
+    audio from context ID "A" will be played first.
+
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._contexts_queue = asyncio.Queue()
+        self._contexts: Dict[str, asyncio.Queue] = {}
+        self._audio_context_task = None
+
+    async def create_audio_context(self, context_id: str):
+        """Create a new audio context."""
+        await self._contexts_queue.put(context_id)
+        self._contexts[context_id] = asyncio.Queue()
+        logger.trace(f"{self} created audio context {context_id}")
+
+    async def append_to_audio_context(self, context_id: str, frame: TTSAudioRawFrame):
+        """Append audio to an existing context."""
+        if self.audio_context_available(context_id):
+            logger.trace(f"{self} appending audio {frame} to audio context {context_id}")
+            await self._contexts[context_id].put(frame)
+        else:
+            logger.warning(f"{self} unable to append audio to context {context_id}")
+
+    async def remove_audio_context(self, context_id: str):
+        """Remove an existing audio context."""
+        if self.audio_context_available(context_id):
+            # We just mark the audio context for deletion by appending
+            # None. Once we reach None while handling audio we know we can
+            # safely remove the context.
+            logger.trace(f"{self} marking audio context {context_id} for deletion")
+            await self._contexts[context_id].put(None)
+        else:
+            logger.warning(f"{self} unable to remove context {context_id}")
+
+    def audio_context_available(self, context_id: str) -> bool:
+        """Checks whether the given audio context is registered."""
+        return context_id in self._contexts
+
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+        self._create_audio_context_task()
+
+    async def stop(self, frame: EndFrame):
+        await super().stop(frame)
+        if self._audio_context_task:
+            # Indicate no more audio contexts are available. this will end the
+            # task cleanly after all contexts have been processed.
+            await self._contexts_queue.put(None)
+            await self.wait_for_task(self._audio_context_task)
+            self._audio_context_task = None
+
+    async def cancel(self, frame: CancelFrame):
+        await super().cancel(frame)
+        await self._stop_audio_context_task()
+
+    async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
+        await super()._handle_interruption(frame, direction)
+        await self._stop_audio_context_task()
+        self._create_audio_context_task()
+
+    def _create_audio_context_task(self):
+        if not self._audio_context_task:
+            self._contexts_queue = asyncio.Queue()
+            self._contexts: Dict[str, asyncio.Queue] = {}
+            self._audio_context_task = self.create_task(self._audio_context_task_handler())
+
+    async def _stop_audio_context_task(self):
+        if self._audio_context_task:
+            await self.cancel_task(self._audio_context_task)
+            self._audio_context_task = None
+
+    async def _audio_context_task_handler(self):
+        """In this task we process audio contexts in order."""
+        running = True
+        while running:
+            context_id = await self._contexts_queue.get()
+
+            if context_id:
+                # Process the audio context until the context doesn't have more
+                # audio available (i.e. we find None).
+                await self._handle_audio_context(context_id)
+
+                # We just finished processing the context, so we can safely remove it.
+                del self._contexts[context_id]
+
+                # Append some silence between sentences.
+                silence = b"\x00" * self.sample_rate
+                frame = TTSAudioRawFrame(
+                    audio=silence, sample_rate=self.sample_rate, num_channels=1
+                )
+                await self.push_frame(frame)
+            else:
+                running = False
+
+            self._contexts_queue.task_done()
+
+    async def _handle_audio_context(self, context_id: str):
+        # If we don't receive any audio during this time, we consider the context finished.
+        AUDIO_CONTEXT_TIMEOUT = 3.0
+        queue = self._contexts[context_id]
+        running = True
+        while running:
+            try:
+                frame = await asyncio.wait_for(queue.get(), timeout=AUDIO_CONTEXT_TIMEOUT)
+                if frame:
+                    await self.push_frame(frame)
+                running = frame is not None
+            except asyncio.TimeoutError:
+                # We didn't get audio, so let's consider this context finished.
+                logger.trace(f"{self} time out on audio context {context_id}")
+                break
+
+
 class STTService(AIService):
     """STTService is a base class for speech-to-text services."""
 
-    def __init__(self, audio_passthrough=False, **kwargs):
+    def __init__(
+        self,
+        audio_passthrough=False,
+        # STT input sample rate
+        sample_rate: Optional[int] = None,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self._audio_passthrough = audio_passthrough
+        self._init_sample_rate = sample_rate
+        self._sample_rate = 0
         self._settings: Dict[str, Any] = {}
         self._muted: bool = False
 
@@ -478,11 +958,13 @@ class STTService(AIService):
         """Returns whether the STT service is currently muted."""
         return self._muted
 
-    @abstractmethod
+    @property
+    def sample_rate(self) -> int:
+        return self._sample_rate
+
     async def set_model(self, model: str):
         self.set_model_name(model)
 
-    @abstractmethod
     async def set_language(self, language: Language):
         pass
 
@@ -491,6 +973,10 @@ class STTService(AIService):
         """Returns transcript as a string"""
         pass
 
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+        self._sample_rate = self._init_sample_rate or frame.audio_in_sample_rate
+
     async def _update_settings(self, settings: Mapping[str, Any]):
         logger.info(f"Updating STT settings: {self._settings}")
         for key, value in settings.items():
@@ -504,9 +990,11 @@ class STTService(AIService):
             else:
                 logger.warning(f"Unknown setting for STT service: {key}")
 
-    async def process_audio_frame(self, frame: AudioRawFrame):
-        if not self._muted:
-            await self.process_generator(self.run_stt(frame.audio))
+    async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection):
+        if self._muted:
+            return
+
+        await self.process_generator(self.run_stt(frame.audio))
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         """Processes a frame of audio data, either buffering or transcribing it."""
@@ -516,7 +1004,7 @@ class STTService(AIService):
             # In this service we accumulate audio internally and at the end we
             # push a TextFrame. We also push audio downstream in case someone
             # else needs it.
-            await self.process_audio_frame(frame)
+            await self.process_audio_frame(frame, direction)
             if self._audio_passthrough:
                 await self.push_frame(frame, direction)
         elif isinstance(frame, STTUpdateSettingsFrame):
@@ -529,74 +1017,64 @@ class STTService(AIService):
 
 
 class SegmentedSTTService(STTService):
-    """SegmentedSTTService is an STTService that will detect speech and will run
-    speech-to-text on speech segments only, instead of a continous stream.
+    """SegmentedSTTService is an STTService that uses VAD events to detect
+    speech and will run speech-to-text on speech segments only, instead of a
+    continous stream. Since it uses VAD it means that VAD needs to be enabled in
+    the pipeline.
+
+    This service always keeps a small audio buffer to take into account that VAD
+    events are delayed from when the user speech really starts.
 
     """
 
-    def __init__(
-        self,
-        *,
-        min_volume: float = 0.6,
-        max_silence_secs: float = 0.3,
-        max_buffer_secs: float = 1.5,
-        sample_rate: int = 24000,
-        num_channels: int = 1,
-        **kwargs,
-    ):
-        super().__init__(**kwargs)
-        self._min_volume = min_volume
-        self._max_silence_secs = max_silence_secs
-        self._max_buffer_secs = max_buffer_secs
-        self._sample_rate = sample_rate
-        self._num_channels = num_channels
-        (self._content, self._wave) = self._new_wave()
-        self._silence_num_frames = 0
-        # Volume exponential smoothing
-        self._smoothing_factor = 0.2
-        self._prev_volume = 0
+    def __init__(self, *, sample_rate: Optional[int] = None, **kwargs):
+        super().__init__(sample_rate=sample_rate, **kwargs)
+        self._content = None
+        self._wave = None
+        self._audio_buffer = bytearray()
+        self._audio_buffer_size_1s = 0
+        self._user_speaking = False
 
-    async def process_audio_frame(self, frame: AudioRawFrame):
-        # Try to filter out empty background noise
-        volume = self._get_smoothed_volume(frame)
-        if volume >= self._min_volume:
-            # If volume is high enough, write new data to wave file
-            self._wave.writeframes(frame.audio)
-            self._silence_num_frames = 0
-        else:
-            self._silence_num_frames += frame.num_frames
-        self._prev_volume = volume
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+        self._audio_buffer_size_1s = self.sample_rate * 2
 
-        # If buffer is not empty and we have enough data or there's been a long
-        # silence, transcribe the audio gathered so far.
-        silence_secs = self._silence_num_frames / self._sample_rate
-        buffer_secs = self._wave.getnframes() / self._sample_rate
-        if self._content.tell() > 0 and (
-            buffer_secs > self._max_buffer_secs or silence_secs > self._max_silence_secs
-        ):
-            self._silence_num_frames = 0
-            self._wave.close()
-            self._content.seek(0)
-            await self.process_generator(self.run_stt(self._content.read()))
-            (self._content, self._wave) = self._new_wave()
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
 
-    async def stop(self, frame: EndFrame):
-        self._wave.close()
+        if isinstance(frame, UserStartedSpeakingFrame):
+            await self._handle_user_started_speaking(frame)
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            await self._handle_user_stopped_speaking(frame)
 
-    async def cancel(self, frame: CancelFrame):
-        self._wave.close()
+    async def _handle_user_started_speaking(self, frame: UserStartedSpeakingFrame):
+        self._user_speaking = True
+
+    async def _handle_user_stopped_speaking(self, frame: UserStoppedSpeakingFrame):
+        self._user_speaking = False
 
-    def _new_wave(self):
         content = io.BytesIO()
-        ww = wave.open(content, "wb")
-        ww.setsampwidth(2)
-        ww.setnchannels(self._num_channels)
-        ww.setframerate(self._sample_rate)
-        return (content, ww)
+        wav = wave.open(content, "wb")
+        wav.setsampwidth(2)
+        wav.setnchannels(1)
+        wav.setframerate(self.sample_rate)
+        wav.writeframes(self._audio_buffer)
+        wav.close()
+        content.seek(0)
 
-    def _get_smoothed_volume(self, frame: AudioRawFrame) -> float:
-        volume = calculate_audio_volume(frame.audio, frame.sample_rate)
-        return exp_smoothing(volume, self._prev_volume, self._smoothing_factor)
+        await self.process_generator(self.run_stt(content.read()))
+
+        # Start clean.
+        self._audio_buffer.clear()
+
+    async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection):
+        # If the user is speaking the audio buffer will keep growin.
+        self._audio_buffer += frame.audio
+
+        # If the user is not speaking we keep just a little bit of audio.
+        if not self._user_speaking and len(self._audio_buffer) > self._audio_buffer_size_1s:
+            discarded = len(self._audio_buffer) - self._audio_buffer_size_1s
+            self._audio_buffer = self._audio_buffer[discarded:]
 
 
 class ImageGenService(AIService):

src/pipecat/services/anthropic.py

@@ -4,34 +4,33 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+import asyncio
 import base64
 import copy
 import io
 import json
 import re
-from asyncio import CancelledError
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Mapping, Optional, Union
 
+import httpx
 from loguru import logger
 from PIL import Image
 from pydantic import BaseModel, Field
 
+from pipecat.adapters.services.anthropic_adapter import AnthropicLLMAdapter
 from pipecat.frames.frames import (
     Frame,
+    FunctionCallCancelFrame,
     FunctionCallInProgressFrame,
     FunctionCallResultFrame,
-    FunctionCallResultProperties,
     LLMEnablePromptCachingFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesFrame,
     LLMTextFrame,
     LLMUpdateSettingsFrame,
-    OpenAILLMContextAssistantTimestampFrame,
-    StartInterruptionFrame,
     UserImageRawFrame,
-    UserImageRequestFrame,
     VisionImageRawFrame,
 )
 from pipecat.metrics.metrics import LLMTokenUsage
@@ -45,7 +44,6 @@ from pipecat.processors.aggregators.openai_llm_context import (
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.ai_services import LLMService
-from pipecat.utils.time import time_now_iso8601
 
 try:
     from anthropic import NOT_GIVEN, AsyncAnthropic, NotGiven
@@ -58,13 +56,6 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
-# internal use only -- todo: refactor
-@dataclass
-class AnthropicImageMessageFrame(Frame):
-    user_image_raw_frame: UserImageRawFrame
-    text: Optional[str] = None
-
-
 @dataclass
 class AnthropicContextAggregatorPair:
     _user: "AnthropicUserContextAggregator"
@@ -84,6 +75,9 @@ class AnthropicLLMService(LLMService):
     use `AsyncAnthropicBedrock` and `AsyncAnthropicVertex` clients
     """
 
+    # Overriding the default adapter to use the Anthropic one.
+    adapter_class = AnthropicLLMAdapter
+
     class InputParams(BaseModel):
         enable_prompt_caching_beta: Optional[bool] = False
         max_tokens: Optional[int] = Field(default_factory=lambda: 4096, ge=1)
@@ -96,7 +90,7 @@ class AnthropicLLMService(LLMService):
         self,
         *,
         api_key: str,
-        model: str = "claude-3-5-sonnet-20241022",
+        model: str = "claude-3-7-sonnet-20250219",
         params: InputParams = InputParams(),
         client=None,
         **kwargs,
@@ -122,14 +116,38 @@ class AnthropicLLMService(LLMService):
     def enable_prompt_caching_beta(self) -> bool:
         return self._enable_prompt_caching_beta
 
-    @staticmethod
     def create_context_aggregator(
-        context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True
+        self,
+        context: OpenAILLMContext,
+        *,
+        user_kwargs: Mapping[str, Any] = {},
+        assistant_kwargs: Mapping[str, Any] = {},
     ) -> AnthropicContextAggregatorPair:
-        user = AnthropicUserContextAggregator(context)
-        assistant = AnthropicAssistantContextAggregator(
-            user, expect_stripped_words=assistant_expect_stripped_words
-        )
+        """Create an instance of AnthropicContextAggregatorPair from an
+        OpenAILLMContext. Constructor keyword arguments for both the user and
+        assistant aggregators can be provided.
+
+        Args:
+            context (OpenAILLMContext): The LLM context.
+            user_kwargs (Mapping[str, Any], optional): Additional keyword
+                arguments for the user context aggregator constructor. Defaults
+                to an empty mapping.
+            assistant_kwargs (Mapping[str, Any], optional): Additional keyword
+                arguments for the assistant context aggregator
+                constructor. Defaults to an empty mapping.
+
+        Returns:
+            AnthropicContextAggregatorPair: A pair of context aggregators, one
+            for the user and one for the assistant, encapsulated in an
+            AnthropicContextAggregatorPair.
+
+        """
+        context.set_llm_adapter(self.get_llm_adapter())
+
+        if isinstance(context, OpenAILLMContext):
+            context = AnthropicLLMContext.from_openai_context(context)
+        user = AnthropicUserContextAggregator(context, **user_kwargs)
+        assistant = AnthropicAssistantContextAggregator(context, **assistant_kwargs)
         return AnthropicContextAggregatorPair(_user=user, _assistant=assistant)
 
     async def _process_context(self, context: OpenAILLMContext):
@@ -149,7 +167,7 @@ class AnthropicLLMService(LLMService):
             await self.start_processing_metrics()
 
             logger.debug(
-                f"Generating chat: {context.system} | {context.get_messages_for_logging()}"
+                f"{self}: Generating chat [{context.system}] | [{context.get_messages_for_logging()}]"
             )
 
             messages = context.messages
@@ -249,12 +267,14 @@ class AnthropicLLMService(LLMService):
                     if total_input_tokens >= 1024:
                         context.turns_above_cache_threshold += 1
 
-        except CancelledError:
+        except asyncio.CancelledError:
             # If we're interrupted, we won't get a complete usage report. So set our flag to use the
             # token estimate. The reraise the exception so all the processors running in this task
             # also get cancelled.
             use_completion_tokens_estimate = True
             raise
+        except httpx.TimeoutException:
+            await self._call_event_handler("on_completion_timeout")
         except Exception as e:
             logger.exception(f"{self} exception: {e}")
         finally:
@@ -326,9 +346,9 @@ class AnthropicLLMService(LLMService):
 class AnthropicLLMContext(OpenAILLMContext):
     def __init__(
         self,
-        messages: list[dict] | None = None,
-        tools: list[dict] | None = None,
-        tool_choice: dict | None = None,
+        messages: Optional[List[dict]] = None,
+        tools: Optional[List[dict]] = None,
+        tool_choice: Optional[dict] = None,
         *,
         system: Union[str, NotGiven] = NOT_GIVEN,
     ):
@@ -357,6 +377,7 @@ class AnthropicLLMContext(OpenAILLMContext):
             tools=openai_context.tools,
             tool_choice=openai_context.tool_choice,
         )
+        self.set_llm_adapter(openai_context.get_llm_adapter())
         self._restructure_from_openai_messages()
         return self
 
@@ -651,45 +672,7 @@ class AnthropicLLMContext(OpenAILLMContext):
 
 
 class AnthropicUserContextAggregator(LLMUserContextAggregator):
-    def __init__(self, context: OpenAILLMContext | AnthropicLLMContext):
-        super().__init__(context=context)
-
-        if isinstance(context, OpenAILLMContext):
-            self._context = AnthropicLLMContext.from_openai_context(context)
-
-    async def process_frame(self, frame, direction):
-        await super().process_frame(frame, direction)
-        # Our parent method has already called push_frame(). So we can't interrupt the
-        # flow here and we don't need to call push_frame() ourselves. Possibly something
-        # to talk through (tagging @aleix). At some point we might need to refactor these
-        # context aggregators.
-        try:
-            if isinstance(frame, UserImageRequestFrame):
-                # The LLM sends a UserImageRequestFrame upstream. Cache any context provided with
-                # that frame so we can use it when we assemble the image message in the assistant
-                # context aggregator.
-                if frame.context:
-                    if isinstance(frame.context, str):
-                        self._context._user_image_request_context[frame.user_id] = frame.context
-                    else:
-                        logger.error(
-                            f"Unexpected UserImageRequestFrame context type: {type(frame.context)}"
-                        )
-                        del self._context._user_image_request_context[frame.user_id]
-                else:
-                    if frame.user_id in self._context._user_image_request_context:
-                        del self._context._user_image_request_context[frame.user_id]
-            elif isinstance(frame, UserImageRawFrame):
-                # Push a new AnthropicImageMessageFrame with the text context we cached
-                # downstream to be handled by our assistant context aggregator. This is
-                # necessary so that we add the message to the context in the right order.
-                text = self._context._user_image_request_context.get(frame.user_id) or ""
-                if text:
-                    del self._context._user_image_request_context[frame.user_id]
-                frame = AnthropicImageMessageFrame(user_image_raw_frame=frame, text=text)
-                await self.push_frame(frame)
-        except Exception as e:
-            logger.error(f"Error processing frame: {e}")
+    pass
 
 
 #
@@ -703,115 +686,64 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator):
 
 
 class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
-    def __init__(self, user_context_aggregator: AnthropicUserContextAggregator, **kwargs):
-        super().__init__(context=user_context_aggregator._context, **kwargs)
-        self._user_context_aggregator = user_context_aggregator
-        self._function_call_in_progress = None
-        self._function_call_result = None
-        self._pending_image_frame_message = None
+    async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
+        assistant_message = {"role": "assistant", "content": []}
+        assistant_message["content"].append(
+            {
+                "type": "tool_use",
+                "id": frame.tool_call_id,
+                "name": frame.function_name,
+                "input": frame.arguments,
+            }
+        )
+        self._context.add_message(assistant_message)
+        self._context.add_message(
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": frame.tool_call_id,
+                        "content": "IN_PROGRESS",
+                    }
+                ],
+            }
+        )
 
-    async def process_frame(self, frame, direction):
-        await super().process_frame(frame, direction)
-        # See note above about not calling push_frame() here.
-        if isinstance(frame, StartInterruptionFrame):
-            self._function_call_in_progress = None
-            self._function_call_finished = None
-        elif isinstance(frame, FunctionCallInProgressFrame):
-            self._function_call_in_progress = frame
-        elif isinstance(frame, FunctionCallResultFrame):
-            if (
-                self._function_call_in_progress
-                and self._function_call_in_progress.tool_call_id == frame.tool_call_id
-            ):
-                self._function_call_in_progress = None
-                self._function_call_result = frame
-                await self._push_aggregation()
-            else:
-                logger.warning(
-                    "FunctionCallResultFrame tool_call_id != InProgressFrame tool_call_id"
-                )
-                self._function_call_in_progress = None
-                self._function_call_result = None
-        elif isinstance(frame, AnthropicImageMessageFrame):
-            self._pending_image_frame_message = frame
-            await self._push_aggregation()
+    async def handle_function_call_result(self, frame: FunctionCallResultFrame):
+        if frame.result:
+            result = json.dumps(frame.result)
+            await self._update_function_call_result(frame.function_name, frame.tool_call_id, result)
+        else:
+            await self._update_function_call_result(
+                frame.function_name, frame.tool_call_id, "COMPLETED"
+            )
 
-    async def _push_aggregation(self):
-        if not (
-            self._aggregation or self._function_call_result or self._pending_image_frame_message
-        ):
-            return
+    async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame):
+        await self._update_function_call_result(
+            frame.function_name, frame.tool_call_id, "CANCELLED"
+        )
 
-        run_llm = False
-        properties: Optional[FunctionCallResultProperties] = None
+    async def _update_function_call_result(
+        self, function_name: str, tool_call_id: str, result: Any
+    ):
+        for message in self._context.messages:
+            if message["role"] == "user":
+                for content in message["content"]:
+                    if (
+                        isinstance(content, dict)
+                        and content["type"] == "tool_result"
+                        and content["tool_use_id"] == tool_call_id
+                    ):
+                        content["content"] = result
 
-        aggregation = self._aggregation
-        self._reset()
-
-        try:
-            if self._function_call_result:
-                frame = self._function_call_result
-                properties = frame.properties
-                self._function_call_result = None
-                if frame.result:
-                    assistant_message = {"role": "assistant", "content": []}
-                    if aggregation:
-                        assistant_message["content"].append({"type": "text", "text": aggregation})
-                    assistant_message["content"].append(
-                        {
-                            "type": "tool_use",
-                            "id": frame.tool_call_id,
-                            "name": frame.function_name,
-                            "input": frame.arguments,
-                        }
-                    )
-                    self._context.add_message(assistant_message)
-                    self._context.add_message(
-                        {
-                            "role": "user",
-                            "content": [
-                                {
-                                    "type": "tool_result",
-                                    "tool_use_id": frame.tool_call_id,
-                                    "content": json.dumps(frame.result),
-                                }
-                            ],
-                        }
-                    )
-                    if properties and properties.run_llm is not None:
-                        # If the tool call result has a run_llm property, use it
-                        run_llm = properties.run_llm
-                    else:
-                        # Default behavior
-                        run_llm = True
-            elif aggregation:
-                self._context.add_message({"role": "assistant", "content": aggregation})
-
-            if self._pending_image_frame_message:
-                frame = self._pending_image_frame_message
-                self._pending_image_frame_message = None
-                self._context.add_image_frame_message(
-                    format=frame.user_image_raw_frame.format,
-                    size=frame.user_image_raw_frame.size,
-                    image=frame.user_image_raw_frame.image,
-                    text=frame.text,
-                )
-                run_llm = True
-
-            if run_llm:
-                await self._user_context_aggregator.push_context_frame()
-
-            # Emit the on_context_updated callback once the function call result is added to the context
-            if properties and properties.on_context_updated is not None:
-                await properties.on_context_updated()
-
-            # Push context frame
-            frame = OpenAILLMContextFrame(self._context)
-            await self.push_frame(frame)
-
-            # Push timestamp frame with current time
-            timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601())
-            await self.push_frame(timestamp_frame)
-
-        except Exception as e:
-            logger.error(f"Error processing frame: {e}")
+    async def handle_user_image_frame(self, frame: UserImageRawFrame):
+        await self._update_function_call_result(
+            frame.request.function_name, frame.request.tool_call_id, "COMPLETED"
+        )
+        self._context.add_image_frame_message(
+            format=frame.format,
+            size=frame.size,
+            image=frame.image,
+            text=frame.request.context,
+        )

src/pipecat/services/assemblyai.py

@@ -5,7 +5,7 @@
 #
 
 import asyncio
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 
 from loguru import logger
 
@@ -38,20 +38,17 @@ class AssemblyAISTTService(STTService):
         self,
         *,
         api_key: str,
-        sample_rate: int = 16000,
+        sample_rate: Optional[int] = None,
         encoding: AudioEncoding = AudioEncoding("pcm_s16le"),
         language=Language.EN,  # Only English is supported for Realtime
         **kwargs,
     ):
-        super().__init__(**kwargs)
+        super().__init__(sample_rate=sample_rate, **kwargs)
 
         aai.settings.api_key = api_key
-        self._transcriber: aai.RealtimeTranscriber | None = None
-        # Store reference to the main event loop for use in callback functions
-        self._loop = asyncio.get_event_loop()
+        self._transcriber: Optional[aai.RealtimeTranscriber] = None
 
         self._settings = {
-            "sample_rate": sample_rate,
             "encoding": encoding,
             "language": language,
         }
@@ -94,6 +91,9 @@ class AssemblyAISTTService(STTService):
         AssemblyAI transcriber.
         """
 
+        if self._transcriber:
+            return
+
         def on_open(session_opened: aai.RealtimeSessionOpened):
             """Callback for when the connection to AssemblyAI is opened."""
             logger.info(f"{self}: Connected to AssemblyAI")
@@ -121,7 +121,7 @@ class AssemblyAISTTService(STTService):
 
             # Schedule the coroutine to run in the main event loop
             # This is necessary because this callback runs in a different thread
-            asyncio.run_coroutine_threadsafe(self.push_frame(frame), self._loop)
+            asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
 
         def on_error(error: aai.RealtimeError):
             """Callback for handling errors from AssemblyAI.
@@ -131,14 +131,16 @@ class AssemblyAISTTService(STTService):
             """
             logger.error(f"{self}: An error occurred: {error}")
             # Schedule the coroutine to run in the main event loop
-            asyncio.run_coroutine_threadsafe(self.push_frame(ErrorFrame(str(error))), self._loop)
+            asyncio.run_coroutine_threadsafe(
+                self.push_frame(ErrorFrame(str(error))), self.get_event_loop()
+            )
 
         def on_close():
             """Callback for when the connection to AssemblyAI is closed."""
             logger.info(f"{self}: Disconnected from AssemblyAI")
 
         self._transcriber = aai.RealtimeTranscriber(
-            sample_rate=self._settings["sample_rate"],
+            sample_rate=self.sample_rate,
             encoding=self._settings["encoding"],
             on_data=on_data,
             on_error=on_error,

src/pipecat/services/aws.py

@@ -32,7 +32,7 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
-def language_to_aws_language(language: Language) -> str | None:
+def language_to_aws_language(language: Language) -> Optional[str]:
     language_map = {
         # Arabic
         Language.AR: "arb",
@@ -124,7 +124,7 @@ class PollyTTSService(TTSService):
         aws_session_token: Optional[str] = None,
         region: Optional[str] = None,
         voice_id: str = "Joanna",
-        sample_rate: int = 24000,
+        sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
     ):
@@ -138,7 +138,6 @@ class PollyTTSService(TTSService):
             region_name=region,
         )
         self._settings = {
-            "sample_rate": sample_rate,
             "engine": params.engine,
             "language": self.language_to_service_language(params.language)
             if params.language
@@ -155,7 +154,7 @@ class PollyTTSService(TTSService):
     def can_generate_metrics(self) -> bool:
         return True
 
-    def language_to_service_language(self, language: Language) -> str | None:
+    def language_to_service_language(self, language: Language) -> Optional[str]:
         return language_to_aws_language(language)
 
     def _construct_ssml(self, text: str) -> str:
@@ -198,7 +197,7 @@ class PollyTTSService(TTSService):
                 return audio_data
             return None
 
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
             await self.start_ttfb_metrics()
@@ -226,9 +225,7 @@ class PollyTTSService(TTSService):
                 yield None
                 return
 
-            audio_data = await self._resampler.resample(
-                audio_data, 16000, self._settings["sample_rate"]
-            )
+            audio_data = await self._resampler.resample(audio_data, 16000, self.sample_rate)
 
             await self.start_tts_usage_metrics(text)
 
@@ -239,7 +236,7 @@ class PollyTTSService(TTSService):
                 chunk = audio_data[i : i + chunk_size]
                 if len(chunk) > 0:
                     await self.stop_ttfb_metrics()
-                    frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
+                    frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
                     yield frame
 
             yield TTSStoppedFrame()
@@ -251,16 +248,3 @@ class PollyTTSService(TTSService):
 
         finally:
             yield TTSStoppedFrame()
-
-
-class AWSTTSService(PollyTTSService):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-        import warnings
-
-        with warnings.catch_warnings():
-            warnings.simplefilter("always")
-            warnings.warn(
-                "'AWSTTSService' is deprecated, use 'PollyTTSService' instead.", DeprecationWarning
-            )

src/pipecat/services/azure.py

@@ -10,6 +10,7 @@ from typing import AsyncGenerator, Optional
 
 import aiohttp
 from loguru import logger
+from openai import AsyncAzureOpenAI
 from PIL import Image
 from pydantic import BaseModel
 
@@ -48,7 +49,6 @@ try:
         PushAudioInputStream,
     )
     from azure.cognitiveservices.speech.dialog import AudioConfig
-    from openai import AsyncAzureOpenAI
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
@@ -57,7 +57,7 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
-def language_to_azure_language(language: Language) -> str | None:
+def language_to_azure_language(language: Language) -> Optional[str]:
     language_map = {
         # Afrikaans
         Language.AF: "af-ZA",
@@ -450,14 +450,13 @@ class AzureBaseTTSService(TTSService):
         api_key: str,
         region: str,
         voice="en-US-SaraNeural",
-        sample_rate: int = 24000,
+        sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
     ):
         super().__init__(sample_rate=sample_rate, **kwargs)
 
         self._settings = {
-            "sample_rate": sample_rate,
             "emphasis": params.emphasis,
             "language": self.language_to_service_language(params.language)
             if params.language
@@ -478,7 +477,7 @@ class AzureBaseTTSService(TTSService):
     def can_generate_metrics(self) -> bool:
         return True
 
-    def language_to_service_language(self, language: Language) -> str | None:
+    def language_to_service_language(self, language: Language) -> Optional[str]:
         return language_to_azure_language(language)
 
     def _construct_ssml(self, text: str) -> str:
@@ -530,25 +529,36 @@ class AzureBaseTTSService(TTSService):
 class AzureTTSService(AzureBaseTTSService):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        self._speech_config = None
+        self._speech_synthesizer = None
+        self._audio_queue = asyncio.Queue()
 
-        speech_config = SpeechConfig(
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+
+        if self._speech_config:
+            return
+
+        # Now self.sample_rate is properly initialized
+        self._speech_config = SpeechConfig(
             subscription=self._api_key,
             region=self._region,
             speech_recognition_language=self._settings["language"],
         )
-        speech_config.set_speech_synthesis_output_format(
-            sample_rate_to_output_format(self._settings["sample_rate"])
+        self._speech_config.set_speech_synthesis_output_format(
+            sample_rate_to_output_format(self.sample_rate)
         )
-        speech_config.set_service_property(
+        self._speech_config.set_service_property(
             "synthesizer.synthesis.connection.synthesisConnectionImpl",
             "websocket",
             ServicePropertyChannel.UriQueryParameter,
         )
 
-        self._speech_synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=None)
+        self._speech_synthesizer = SpeechSynthesizer(
+            speech_config=self._speech_config, audio_config=None
+        )
 
         # Set up event handlers
-        self._audio_queue = asyncio.Queue()
         self._speech_synthesizer.synthesizing.connect(self._handle_synthesizing)
         self._speech_synthesizer.synthesis_completed.connect(self._handle_completed)
         self._speech_synthesizer.synthesis_canceled.connect(self._handle_canceled)
@@ -567,58 +577,78 @@ class AzureTTSService(AzureBaseTTSService):
         logger.error(f"Speech synthesis canceled: {evt.result.cancellation_details.reason}")
         self._audio_queue.put_nowait(None)
 
+    async def flush_audio(self):
+        logger.trace(f"{self}: flushing audio")
+
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
-            await self.start_ttfb_metrics()
-            yield TTSStartedFrame()
+            if self._speech_synthesizer is None:
+                error_msg = "Speech synthesizer not initialized."
+                logger.error(error_msg)
+                yield ErrorFrame(error_msg)
+                return
 
-            ssml = self._construct_ssml(text)
+            try:
+                await self.start_ttfb_metrics()
+                yield TTSStartedFrame()
 
-            # Start synthesis
-            self._speech_synthesizer.speak_ssml_async(ssml)
+                ssml = self._construct_ssml(text)
+                self._speech_synthesizer.speak_ssml_async(ssml)
+                await self.start_tts_usage_metrics(text)
 
-            await self.start_tts_usage_metrics(text)
+                # Stream audio chunks as they arrive
+                while True:
+                    chunk = await self._audio_queue.get()
+                    if chunk is None:  # End of stream
+                        break
 
-            # Stream audio chunks as they arrive
-            while True:
-                chunk = await self._audio_queue.get()
-                if chunk is None:  # End of stream
-                    break
+                    await self.stop_ttfb_metrics()
+                    yield TTSAudioRawFrame(
+                        audio=chunk,
+                        sample_rate=self.sample_rate,
+                        num_channels=1,
+                    )
 
-                await self.stop_ttfb_metrics()
+                yield TTSStoppedFrame()
 
-                yield TTSAudioRawFrame(
-                    audio=chunk,
-                    sample_rate=self._settings["sample_rate"],
-                    num_channels=1,
-                )
-
-            yield TTSStoppedFrame()
+            except Exception as e:
+                logger.error(f"{self} error during synthesis: {e}")
+                yield TTSStoppedFrame()
+                # Could add reconnection logic here if needed
+                return
 
         except Exception as e:
-            logger.error(f"{self} error generating TTS: {e}")
-            yield ErrorFrame(f"{self} error: {str(e)}")
+            logger.error(f"{self} exception: {e}")
 
 
 class AzureHttpTTSService(AzureBaseTTSService):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        self._speech_config = None
+        self._speech_synthesizer = None
 
-        speech_config = SpeechConfig(
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+
+        if self._speech_config:
+            return
+
+        self._speech_config = SpeechConfig(
             subscription=self._api_key,
             region=self._region,
             speech_recognition_language=self._settings["language"],
         )
-        speech_config.set_speech_synthesis_output_format(
-            sample_rate_to_output_format(self._settings["sample_rate"])
+        self._speech_config.set_speech_synthesis_output_format(
+            sample_rate_to_output_format(self.sample_rate)
+        )
+        self._speech_synthesizer = SpeechSynthesizer(
+            speech_config=self._speech_config, audio_config=None
         )
 
-        self._speech_synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=None)
-
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         await self.start_ttfb_metrics()
 
@@ -633,7 +663,7 @@ class AzureHttpTTSService(AzureBaseTTSService):
             # Azure always sends a 44-byte header. Strip it off.
             yield TTSAudioRawFrame(
                 audio=result.audio_data[44:],
-                sample_rate=self._settings["sample_rate"],
+                sample_rate=self.sample_rate,
                 num_channels=1,
             )
             yield TTSStoppedFrame()
@@ -650,44 +680,62 @@ class AzureSTTService(STTService):
         *,
         api_key: str,
         region: str,
-        language=Language.EN_US,
-        sample_rate=24000,
-        channels=1,
+        language: Language = Language.EN_US,
+        sample_rate: Optional[int] = None,
         **kwargs,
     ):
-        super().__init__(**kwargs)
+        super().__init__(sample_rate=sample_rate, **kwargs)
 
-        speech_config = SpeechConfig(subscription=api_key, region=region)
-        speech_config.speech_recognition_language = language
-
-        stream_format = AudioStreamFormat(samples_per_second=sample_rate, channels=channels)
-        self._audio_stream = PushAudioInputStream(stream_format)
-
-        audio_config = AudioConfig(stream=self._audio_stream)
-        self._speech_recognizer = SpeechRecognizer(
-            speech_config=speech_config, audio_config=audio_config
+        self._speech_config = SpeechConfig(
+            subscription=api_key,
+            region=region,
+            speech_recognition_language=language_to_azure_language(language),
         )
-        self._speech_recognizer.recognized.connect(self._on_handle_recognized)
+
+        self._audio_stream = None
+        self._speech_recognizer = None
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
         await self.start_processing_metrics()
-        self._audio_stream.write(audio)
+        if self._audio_stream:
+            self._audio_stream.write(audio)
         await self.stop_processing_metrics()
         yield None
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+
+        if self._audio_stream:
+            return
+
+        stream_format = AudioStreamFormat(samples_per_second=self.sample_rate, channels=1)
+        self._audio_stream = PushAudioInputStream(stream_format)
+
+        audio_config = AudioConfig(stream=self._audio_stream)
+
+        self._speech_recognizer = SpeechRecognizer(
+            speech_config=self._speech_config, audio_config=audio_config
+        )
+        self._speech_recognizer.recognized.connect(self._on_handle_recognized)
         self._speech_recognizer.start_continuous_recognition_async()
 
     async def stop(self, frame: EndFrame):
         await super().stop(frame)
-        self._speech_recognizer.stop_continuous_recognition_async()
-        self._audio_stream.close()
+
+        if self._speech_recognizer:
+            self._speech_recognizer.stop_continuous_recognition_async()
+
+        if self._audio_stream:
+            self._audio_stream.close()
 
     async def cancel(self, frame: CancelFrame):
         await super().cancel(frame)
-        self._speech_recognizer.stop_continuous_recognition_async()
-        self._audio_stream.close()
+
+        if self._speech_recognizer:
+            self._speech_recognizer.stop_continuous_recognition_async()
+
+        if self._audio_stream:
+            self._audio_stream.close()
 
     def _on_handle_recognized(self, event):
         if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0:

src/pipecat/services/base_whisper.py

@@ -0,0 +1,184 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from typing import AsyncGenerator, Optional
+
+from loguru import logger
+from openai import AsyncOpenAI
+from openai.types.audio import Transcription
+
+from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
+from pipecat.services.ai_services import SegmentedSTTService
+from pipecat.transcriptions.language import Language
+from pipecat.utils.time import time_now_iso8601
+
+
+def language_to_whisper_language(language: Language) -> Optional[str]:
+    """Language support for Whisper API.
+
+    Docs: https://platform.openai.com/docs/guides/speech-to-text#supported-languages
+    """
+    BASE_LANGUAGES = {
+        Language.AF: "af",
+        Language.AR: "ar",
+        Language.HY: "hy",
+        Language.AZ: "az",
+        Language.BE: "be",
+        Language.BS: "bs",
+        Language.BG: "bg",
+        Language.CA: "ca",
+        Language.ZH: "zh",
+        Language.HR: "hr",
+        Language.CS: "cs",
+        Language.DA: "da",
+        Language.NL: "nl",
+        Language.EN: "en",
+        Language.ET: "et",
+        Language.FI: "fi",
+        Language.FR: "fr",
+        Language.GL: "gl",
+        Language.DE: "de",
+        Language.EL: "el",
+        Language.HE: "he",
+        Language.HI: "hi",
+        Language.HU: "hu",
+        Language.IS: "is",
+        Language.ID: "id",
+        Language.IT: "it",
+        Language.JA: "ja",
+        Language.KN: "kn",
+        Language.KK: "kk",
+        Language.KO: "ko",
+        Language.LV: "lv",
+        Language.LT: "lt",
+        Language.MK: "mk",
+        Language.MS: "ms",
+        Language.MR: "mr",
+        Language.MI: "mi",
+        Language.NE: "ne",
+        Language.NO: "no",
+        Language.FA: "fa",
+        Language.PL: "pl",
+        Language.PT: "pt",
+        Language.RO: "ro",
+        Language.RU: "ru",
+        Language.SR: "sr",
+        Language.SK: "sk",
+        Language.SL: "sl",
+        Language.ES: "es",
+        Language.SW: "sw",
+        Language.SV: "sv",
+        Language.TL: "tl",
+        Language.TA: "ta",
+        Language.TH: "th",
+        Language.TR: "tr",
+        Language.UK: "uk",
+        Language.UR: "ur",
+        Language.VI: "vi",
+        Language.CY: "cy",
+    }
+
+    result = BASE_LANGUAGES.get(language)
+
+    # If not found in base languages, try to find the base language from a variant
+    if not result:
+        lang_str = str(language.value)
+        base_code = lang_str.split("-")[0].lower()
+        result = base_code if base_code in BASE_LANGUAGES.values() else None
+
+    return result
+
+
+class BaseWhisperSTTService(SegmentedSTTService):
+    """Base class for Whisper-based speech-to-text services.
+
+    Provides common functionality for services implementing the Whisper API interface,
+    including metrics generation and error handling.
+
+    Args:
+        model: Name of the Whisper model to use.
+        api_key: Service API key. Defaults to None.
+        base_url: Service API base URL. Defaults to None.
+        language: Language of the audio input. Defaults to English.
+        prompt: Optional text to guide the model's style or continue a previous segment.
+        temperature: Sampling temperature between 0 and 1. Defaults to 0.0.
+        **kwargs: Additional arguments passed to SegmentedSTTService.
+    """
+
+    def __init__(
+        self,
+        *,
+        model: str,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        language: Optional[Language] = Language.EN,
+        prompt: Optional[str] = None,
+        temperature: Optional[float] = None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.set_model_name(model)
+        self._client = self._create_client(api_key, base_url)
+        self._language = self.language_to_service_language(language or Language.EN)
+        self._prompt = prompt
+        self._temperature = temperature
+
+    def _create_client(self, api_key: Optional[str], base_url: Optional[str]):
+        return AsyncOpenAI(api_key=api_key, base_url=base_url)
+
+    async def set_model(self, model: str):
+        self.set_model_name(model)
+
+    def can_generate_metrics(self) -> bool:
+        return True
+
+    def language_to_service_language(self, language: Language) -> Optional[str]:
+        return language_to_whisper_language(language)
+
+    async def set_language(self, language: Language):
+        """Set the language for transcription.
+
+        Args:
+            language: The Language enum value to use for transcription.
+        """
+        logger.info(f"Switching STT language to: [{language}]")
+        self._language = language
+
+    async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
+        try:
+            await self.start_processing_metrics()
+            await self.start_ttfb_metrics()
+
+            response = await self._transcribe(audio)
+
+            await self.stop_ttfb_metrics()
+            await self.stop_processing_metrics()
+
+            text = response.text.strip()
+
+            if text:
+                logger.debug(f"Transcription: [{text}]")
+                yield TranscriptionFrame(text, "", time_now_iso8601())
+            else:
+                logger.warning("Received empty transcription from API")
+
+        except Exception as e:
+            logger.exception(f"Exception during transcription: {e}")
+            yield ErrorFrame(f"Error during transcription: {str(e)}")
+
+    async def _transcribe(self, audio: bytes) -> Transcription:
+        """Transcribe audio data to text.
+
+        Args:
+            audio: Raw audio data in WAV format.
+
+        Returns:
+            Transcription: Object containing the transcribed text.
+
+        Raises:
+            NotImplementedError: Must be implemented by subclasses.
+        """
+        raise NotImplementedError

src/pipecat/services/canonical.py

@@ -9,7 +9,7 @@ import os
 import uuid
 import wave
 from datetime import datetime
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple
 
 import aiohttp
 from loguru import logger
@@ -62,17 +62,21 @@ class CanonicalMetricsService(AIService):
         self,
         *,
         aiohttp_session: aiohttp.ClientSession,
-        audio_buffer_processor: AudioBufferProcessor,
         call_id: str,
         assistant: str,
         api_key: str,
         api_url: str = "https://voiceapp.canonical.chat/api/v1",
         assistant_speaks_first: bool = True,
         output_dir: str = "recordings",
-        context: OpenAILLMContext | None = None,
+        audio_buffer_processor: Optional[AudioBufferProcessor] = None,
+        context: Optional[OpenAILLMContext] = None,
         **kwargs,
     ):
         super().__init__(**kwargs)
+        # Validate that at least one of audio_buffer_processor or context is provided
+        if audio_buffer_processor is None and context is None:
+            raise ValueError("At least one of audio_buffer_processor or context must be specified")
+
         self._aiohttp_session = aiohttp_session
         self._audio_buffer_processor = audio_buffer_processor
         self._api_key = api_key
@@ -85,16 +89,36 @@ class CanonicalMetricsService(AIService):
 
     async def stop(self, frame: EndFrame):
         await super().stop(frame)
-        await self._process_audio()
+        await self._process_completion()
 
     async def cancel(self, frame: CancelFrame):
         await super().cancel(frame)
-        await self._process_audio()
+        await self._process_completion()
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
         await self.push_frame(frame, direction)
 
+    async def _process_completion(self):
+        if self._audio_buffer_processor is not None:
+            await self._process_audio()
+        elif self._context is not None:
+            await self._process_transcript()
+
+    async def _process_transcript(self):
+        params = {
+            "callId": self._call_id,
+            "assistant": {"id": self._assistant, "speaksFirst": self._assistant_speaks_first},
+            "transcript": self._context.messages,
+        }
+        response = await self._aiohttp_session.post(
+            f"{self._api_url}/call",
+            headers=self._request_headers(),
+            json=params,
+        )
+        if not response.ok:
+            logger.error(f"Failed to process transcript: {await response.text()}")
+
     async def _process_audio(self):
         audio_buffer_processor = self._audio_buffer_processor
 

src/pipecat/services/cartesia.py

@@ -13,23 +13,21 @@ from loguru import logger
 from pydantic import BaseModel
 
 from pipecat.frames.frames import (
-    BotStoppedSpeakingFrame,
     CancelFrame,
     EndFrame,
     ErrorFrame,
     Frame,
-    LLMFullResponseEndFrame,
     StartFrame,
     StartInterruptionFrame,
     TTSAudioRawFrame,
-    TTSSpeakFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.ai_services import TTSService, WordTTSService
-from pipecat.services.websocket_service import WebsocketService
+from pipecat.services.ai_services import AudioContextWordTTSService, TTSService
 from pipecat.transcriptions.language import Language
+from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
+from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator
 
 # See .env.example for Cartesia configuration needed
 try:
@@ -43,7 +41,7 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
-def language_to_cartesia_language(language: Language) -> str | None:
+def language_to_cartesia_language(language: Language) -> Optional[str]:
     BASE_LANGUAGES = {
         Language.DE: "de",
         Language.EN: "en",
@@ -75,7 +73,7 @@ def language_to_cartesia_language(language: Language) -> str | None:
     return result
 
 
-class CartesiaTTSService(WordTTSService, WebsocketService):
+class CartesiaTTSService(AudioContextWordTTSService):
     class InputParams(BaseModel):
         language: Optional[Language] = Language.EN
         speed: Optional[Union[str, float]] = ""
@@ -88,11 +86,12 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
         voice_id: str,
         cartesia_version: str = "2024-06-10",
         url: str = "wss://api.cartesia.ai/tts/websocket",
-        model: str = "sonic",
-        sample_rate: int = 24000,
+        model: str = "sonic-2",
+        sample_rate: Optional[int] = None,
         encoding: str = "pcm_s16le",
         container: str = "raw",
         params: InputParams = InputParams(),
+        text_aggregator: Optional[BaseTextAggregator] = None,
         **kwargs,
     ):
         # Aggregating sentences still gives cleaner-sounding results and fewer
@@ -105,14 +104,14 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
         # if we're interrupted. Cartesia gives us word-by-word timestamps. We
         # can use those to generate text frames ourselves aligned with the
         # playout timing of the audio!
-        WordTTSService.__init__(
-            self,
+        super().__init__(
             aggregate_sentences=True,
             push_text_frames=False,
+            pause_frame_processing=True,
             sample_rate=sample_rate,
+            text_aggregator=text_aggregator or SkipTagsAggregator([("<spell>", "</spell>")]),
             **kwargs,
         )
-        WebsocketService.__init__(self)
 
         self._api_key = api_key
         self._cartesia_version = cartesia_version
@@ -121,7 +120,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
             "output_format": {
                 "container": container,
                 "encoding": encoding,
-                "sample_rate": sample_rate,
+                "sample_rate": 0,
             },
             "language": self.language_to_service_language(params.language)
             if params.language
@@ -143,7 +142,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
         await super().set_model(model)
         logger.info(f"Switching TTS model to: [{model}]")
 
-    def language_to_service_language(self, language: Language) -> str | None:
+    def language_to_service_language(self, language: Language) -> Optional[str]:
         return language_to_cartesia_language(language)
 
     def _build_msg(
@@ -174,6 +173,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+        self._settings["output_format"]["sample_rate"] = self.sample_rate
         await self._connect()
 
     async def stop(self, frame: EndFrame):
@@ -186,18 +186,20 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
 
     async def _connect(self):
         await self._connect_websocket()
-
-        self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
+        if not self._receive_task:
+            self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
 
     async def _disconnect(self):
-        await self._disconnect_websocket()
-
         if self._receive_task:
             await self.cancel_task(self._receive_task)
             self._receive_task = None
 
+        await self._disconnect_websocket()
+
     async def _connect_websocket(self):
         try:
+            if self._websocket:
+                return
             logger.debug("Connecting to Cartesia")
             self._websocket = await websockets.connect(
                 f"{self._url}?api_key={self._api_key}&cartesia_version={self._cartesia_version}"
@@ -205,6 +207,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
         except Exception as e:
             logger.error(f"{self} initialization error: {e}")
             self._websocket = None
+            await self._call_event_handler("on_connection_error", f"{e}")
 
     async def _disconnect_websocket(self):
         try:
@@ -238,21 +241,19 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
         logger.trace(f"{self}: flushing audio")
         msg = self._build_msg(text="", continue_transcript=False)
         await self._websocket.send(msg)
+        self._context_id = None
 
     async def _receive_messages(self):
         async for message in self._get_websocket():
             msg = json.loads(message)
-            if not msg or msg["context_id"] != self._context_id:
+            if not msg or not self.audio_context_available(msg["context_id"]):
                 continue
             if msg["type"] == "done":
                 await self.stop_ttfb_metrics()
-                # Unset _context_id but not the _context_id_start_timestamp
-                # because we are likely still playing out audio and need the
-                # timestamp to set send context frames.
-                self._context_id = None
                 await self.add_word_timestamps(
                     [("TTSStoppedFrame", 0), ("LLMFullResponseEndFrame", 0), ("Reset", 0)]
                 )
+                await self.remove_audio_context(msg["context_id"])
             elif msg["type"] == "timestamps":
                 await self.add_word_timestamps(
                     list(zip(msg["word_timestamps"]["words"], msg["word_timestamps"]["start"]))
@@ -262,33 +263,21 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
                 self.start_word_timestamps()
                 frame = TTSAudioRawFrame(
                     audio=base64.b64decode(msg["data"]),
-                    sample_rate=self._settings["output_format"]["sample_rate"],
+                    sample_rate=self.sample_rate,
                     num_channels=1,
                 )
-                await self.push_frame(frame)
+                await self.append_to_audio_context(msg["context_id"], frame)
             elif msg["type"] == "error":
                 logger.error(f"{self} error: {msg}")
                 await self.push_frame(TTSStoppedFrame())
                 await self.stop_all_metrics()
                 await self.push_error(ErrorFrame(f"{self} error: {msg['error']}"))
+                self._context_id = None
             else:
                 logger.error(f"{self} error, unknown message type: {msg}")
 
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        # If we received a TTSSpeakFrame and the LLM response included text (it
-        # might be that it's only a function calling response) we pause
-        # processing more frames until we receive a BotStoppedSpeakingFrame.
-        if isinstance(frame, TTSSpeakFrame):
-            await self.pause_processing_frames()
-        elif isinstance(frame, LLMFullResponseEndFrame) and self._context_id:
-            await self.pause_processing_frames()
-        elif isinstance(frame, BotStoppedSpeakingFrame):
-            await self.resume_processing_frames()
-
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
             if not self._websocket:
@@ -298,6 +287,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
                 await self.start_ttfb_metrics()
                 yield TTSStartedFrame()
                 self._context_id = str(uuid.uuid4())
+                await self.create_audio_context(self._context_id)
 
             msg = self._build_msg(text=text or " ")  # Text must contain at least one character
 
@@ -326,9 +316,9 @@ class CartesiaHttpTTSService(TTSService):
         *,
         api_key: str,
         voice_id: str,
-        model: str = "sonic",
+        model: str = "sonic-2",
         base_url: str = "https://api.cartesia.ai",
-        sample_rate: int = 24000,
+        sample_rate: Optional[int] = None,
         encoding: str = "pcm_s16le",
         container: str = "raw",
         params: InputParams = InputParams(),
@@ -341,7 +331,7 @@ class CartesiaHttpTTSService(TTSService):
             "output_format": {
                 "container": container,
                 "encoding": encoding,
-                "sample_rate": sample_rate,
+                "sample_rate": 0,
             },
             "language": self.language_to_service_language(params.language)
             if params.language
@@ -357,9 +347,13 @@ class CartesiaHttpTTSService(TTSService):
     def can_generate_metrics(self) -> bool:
         return True
 
-    def language_to_service_language(self, language: Language) -> str | None:
+    def language_to_service_language(self, language: Language) -> Optional[str]:
         return language_to_cartesia_language(language)
 
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+        self._settings["output_format"]["sample_rate"] = self.sample_rate
+
     async def stop(self, frame: EndFrame):
         await super().stop(frame)
         await self._client.close()
@@ -369,10 +363,7 @@ class CartesiaHttpTTSService(TTSService):
         await self._client.close()
 
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating TTS: [{text}]")
-
-        await self.start_ttfb_metrics()
-        yield TTSStartedFrame()
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
             voice_controls = None
@@ -383,6 +374,8 @@ class CartesiaHttpTTSService(TTSService):
                 if self._settings["emotion"]:
                     voice_controls["emotion"] = self._settings["emotion"]
 
+            await self.start_ttfb_metrics()
+
             output = await self._client.tts.sse(
                 model_id=self._model_name,
                 transcript=text,
@@ -393,16 +386,17 @@ class CartesiaHttpTTSService(TTSService):
                 _experimental_voice_controls=voice_controls,
             )
 
+            await self.start_tts_usage_metrics(text)
+
+            yield TTSStartedFrame()
+
             frame = TTSAudioRawFrame(
-                audio=output["audio"],
-                sample_rate=self._settings["output_format"]["sample_rate"],
-                num_channels=1,
+                audio=output["audio"], sample_rate=self.sample_rate, num_channels=1
             )
+
             yield frame
         except Exception as e:
             logger.error(f"{self} exception: {e}")
-
-        await self.start_tts_usage_metrics(text)
-
-        await self.stop_ttfb_metrics()
-        yield TTSStoppedFrame()
+        finally:
+            await self.stop_ttfb_metrics()
+            yield TTSStoppedFrame()

src/pipecat/services/cerebras.py

@@ -7,22 +7,12 @@
 from typing import List
 
 from loguru import logger
+from openai import AsyncStream
+from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
 
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai import OpenAILLMService
 
-try:
-    from openai import (
-        AsyncStream,
-    )
-    from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error(
-        "In order to use Cerebras, you need to `pip install pipecat-ai[cerebras]`. Also, set `CEREBRAS_API_KEY` environment variable."
-    )
-    raise Exception(f"Missing module: {e}")
-
 
 class CerebrasLLMService(OpenAILLMService):
     """A service for interacting with Cerebras's API using the OpenAI-compatible interface.

src/pipecat/services/deepgram.py

@@ -5,7 +5,7 @@
 #
 
 import asyncio
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Dict, Optional
 
 from loguru import logger
 
@@ -34,6 +34,7 @@ try:
         AsyncListenWebSocketClient,
         DeepgramClient,
         DeepgramClientOptions,
+        ErrorResponse,
         LiveOptions,
         LiveResultResponse,
         LiveTranscriptionEvents,
@@ -53,14 +54,13 @@ class DeepgramTTSService(TTSService):
         *,
         api_key: str,
         voice: str = "aura-helios-en",
-        sample_rate: int = 24000,
+        sample_rate: Optional[int] = None,
         encoding: str = "linear16",
         **kwargs,
     ):
         super().__init__(sample_rate=sample_rate, **kwargs)
 
         self._settings = {
-            "sample_rate": sample_rate,
             "encoding": encoding,
         }
         self.set_voice(voice)
@@ -70,12 +70,12 @@ class DeepgramTTSService(TTSService):
         return True
 
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         options = SpeakOptions(
             model=self._voice_id,
             encoding=self._settings["encoding"],
-            sample_rate=self._settings["sample_rate"],
+            sample_rate=self.sample_rate,
             container="none",
         )
 
@@ -103,9 +103,7 @@ class DeepgramTTSService(TTSService):
                 chunk = audio_buffer.read(chunk_size)
                 if not chunk:
                     break
-                frame = TTSAudioRawFrame(
-                    audio=chunk, sample_rate=self._settings["sample_rate"], num_channels=1
-                )
+                frame = TTSAudioRawFrame(audio=chunk, sample_rate=self.sample_rate, num_channels=1)
                 yield frame
 
                 yield TTSStoppedFrame()
@@ -121,15 +119,18 @@ class DeepgramSTTService(STTService):
         *,
         api_key: str,
         url: str = "",
-        live_options: LiveOptions = None,
+        sample_rate: Optional[int] = None,
+        live_options: Optional[LiveOptions] = None,
+        addons: Optional[Dict] = None,
         **kwargs,
     ):
-        super().__init__(**kwargs)
+        sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
+        super().__init__(sample_rate=sample_rate, **kwargs)
+
         default_options = LiveOptions(
             encoding="linear16",
             language=Language.EN,
-            model="nova-2-general",
-            sample_rate=16000,
+            model="nova-3-general",
             channels=1,
             interim_results=True,
             smart_format=True,
@@ -149,6 +150,7 @@ class DeepgramSTTService(STTService):
             merged_options.language = merged_options.language.value
 
         self._settings = merged_options.to_dict()
+        self._addons = addons
 
         self._client = DeepgramClient(
             api_key,
@@ -157,13 +159,10 @@ class DeepgramSTTService(STTService):
                 options={"keepalive": "true"},  # verbose=logging.DEBUG
             ),
         )
-        self._connection: AsyncListenWebSocketClient = self._client.listen.asyncwebsocket.v("1")
-        self._connection.on(LiveTranscriptionEvents.Transcript, self._on_message)
+
         if self.vad_enabled:
             self._register_event_handler("on_speech_started")
             self._register_event_handler("on_utterance_end")
-            self._connection.on(LiveTranscriptionEvents.SpeechStarted, self._on_speech_started)
-            self._connection.on(LiveTranscriptionEvents.UtteranceEnd, self._on_utterance_end)
 
     @property
     def vad_enabled(self):
@@ -187,6 +186,7 @@ class DeepgramSTTService(STTService):
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+        self._settings["sample_rate"] = self.sample_rate
         await self._connect()
 
     async def stop(self, frame: EndFrame):
@@ -203,7 +203,25 @@ class DeepgramSTTService(STTService):
 
     async def _connect(self):
         logger.debug("Connecting to Deepgram")
-        if not await self._connection.start(self._settings):
+
+        self._connection: AsyncListenWebSocketClient = self._client.listen.asyncwebsocket.v("1")
+
+        self._connection.on(
+            LiveTranscriptionEvents(LiveTranscriptionEvents.Transcript), self._on_message
+        )
+        self._connection.on(LiveTranscriptionEvents(LiveTranscriptionEvents.Error), self._on_error)
+
+        if self.vad_enabled:
+            self._connection.on(
+                LiveTranscriptionEvents(LiveTranscriptionEvents.SpeechStarted),
+                self._on_speech_started,
+            )
+            self._connection.on(
+                LiveTranscriptionEvents(LiveTranscriptionEvents.UtteranceEnd),
+                self._on_utterance_end,
+            )
+
+        if not await self._connection.start(options=self._settings, addons=self._addons):
             logger.error(f"{self}: unable to connect to Deepgram")
 
     async def _disconnect(self):
@@ -215,6 +233,15 @@ class DeepgramSTTService(STTService):
         await self.start_ttfb_metrics()
         await self.start_processing_metrics()
 
+    async def _on_error(self, *args, **kwargs):
+        error: ErrorResponse = kwargs["error"]
+        logger.warning(f"{self} connection error, will retry: {error}")
+        await self.stop_all_metrics()
+        # NOTE(aleix): we don't disconnect (i.e. call finish on the connection)
+        # because this triggers more errors internally in the Deepgram SDK. So,
+        # we just forget about the previous connection and create a new one.
+        await self._connect()
+
     async def _on_speech_started(self, *args, **kwargs):
         await self.start_metrics()
         await self._call_event_handler("on_speech_started", *args, **kwargs)

src/pipecat/services/deepseek.py

@@ -8,22 +8,12 @@
 from typing import List
 
 from loguru import logger
+from openai import AsyncStream
+from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
 
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai import OpenAILLMService
 
-try:
-    from openai import (
-        AsyncStream,
-    )
-    from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error(
-        "In order to use DeepSeek, you need to `pip install pipecat-ai[deepseek]`. Also, set `DEEPSEEK_API_KEY` environment variable."
-    )
-    raise Exception(f"Missing module: {e}")
-
 
 class DeepSeekLLMService(OpenAILLMService):
     """A service for interacting with DeepSeek's API using the OpenAI-compatible interface.

src/pipecat/services/elevenlabs.py

@@ -14,22 +14,18 @@ from loguru import logger
 from pydantic import BaseModel, model_validator
 
 from pipecat.frames.frames import (
-    BotStoppedSpeakingFrame,
     CancelFrame,
     EndFrame,
     ErrorFrame,
     Frame,
-    LLMFullResponseEndFrame,
     StartFrame,
     StartInterruptionFrame,
     TTSAudioRawFrame,
-    TTSSpeakFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.ai_services import TTSService, WordTTSService
-from pipecat.services.websocket_service import WebsocketService
+from pipecat.services.ai_services import InterruptibleWordTTSService, TTSService
 from pipecat.transcriptions.language import Language
 
 # See .env.example for ElevenLabs configuration needed
@@ -55,7 +51,7 @@ ELEVENLABS_MULTILINGUAL_MODELS = {
 }
 
 
-def language_to_elevenlabs_language(language: Language) -> str | None:
+def language_to_elevenlabs_language(language: Language) -> Optional[str]:
     BASE_LANGUAGES = {
         Language.AR: "ar",
         Language.BG: "bg",
@@ -104,17 +100,60 @@ def language_to_elevenlabs_language(language: Language) -> str | None:
     return result
 
 
-def sample_rate_from_output_format(output_format: str) -> int:
-    match output_format:
-        case "pcm_16000":
-            return 16000
-        case "pcm_22050":
-            return 22050
-        case "pcm_24000":
-            return 24000
-        case "pcm_44100":
-            return 44100
-    return 16000
+def output_format_from_sample_rate(sample_rate: int) -> str:
+    match sample_rate:
+        case 8000:
+            return "pcm_8000"
+        case 16000:
+            return "pcm_16000"
+        case 22050:
+            return "pcm_22050"
+        case 24000:
+            return "pcm_24000"
+        case 44100:
+            return "pcm_44100"
+    logger.warning(
+        f"ElevenLabsTTSService: No output format available for {sample_rate} sample rate"
+    )
+    return "pcm_24000"
+
+
+def build_elevenlabs_voice_settings(
+    settings: Dict[str, Any],
+) -> Optional[Dict[str, Union[float, bool]]]:
+    """Build voice settings dictionary for ElevenLabs based on provided settings.
+
+    Args:
+        settings: Dictionary containing voice settings parameters
+
+    Returns:
+        Dictionary of voice settings or None if required parameters are missing
+    """
+    voice_settings = {}
+    if settings["stability"] is not None and settings["similarity_boost"] is not None:
+        voice_settings["stability"] = settings["stability"]
+        voice_settings["similarity_boost"] = settings["similarity_boost"]
+        if settings["style"] is not None:
+            voice_settings["style"] = settings["style"]
+        if settings["use_speaker_boost"] is not None:
+            voice_settings["use_speaker_boost"] = settings["use_speaker_boost"]
+        if settings["speed"] is not None:
+            voice_settings["speed"] = settings["speed"]
+    else:
+        if settings["style"] is not None:
+            logger.warning(
+                "'style' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
+            )
+        if settings["use_speaker_boost"] is not None:
+            logger.warning(
+                "'use_speaker_boost' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
+            )
+        if settings["speed"] is not None:
+            logger.warning(
+                "'speed' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
+            )
+
+    return voice_settings or None
 
 
 def calculate_word_times(
@@ -138,7 +177,7 @@ def calculate_word_times(
     return word_times
 
 
-class ElevenLabsTTSService(WordTTSService, WebsocketService):
+class ElevenLabsTTSService(InterruptibleWordTTSService):
     class InputParams(BaseModel):
         language: Optional[Language] = None
         optimize_streaming_latency: Optional[str] = None
@@ -146,6 +185,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
         similarity_boost: Optional[float] = None
         style: Optional[float] = None
         use_speaker_boost: Optional[bool] = None
+        speed: Optional[float] = None
         auto_mode: Optional[bool] = True
 
         @model_validator(mode="after")
@@ -165,7 +205,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
         voice_id: str,
         model: str = "eleven_flash_v2_5",
         url: str = "wss://api.elevenlabs.io",
-        output_format: ElevenLabsOutputFormat = "pcm_24000",
+        sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
     ):
@@ -183,34 +223,32 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
         # Finally, ElevenLabs doesn't provide information on when the bot stops
         # speaking for a while, so we want the parent class to send TTSStopFrame
         # after a short period not receiving any audio.
-        WordTTSService.__init__(
-            self,
+        super().__init__(
             aggregate_sentences=True,
             push_text_frames=False,
             push_stop_frames=True,
-            stop_frame_timeout_s=2.0,
-            sample_rate=sample_rate_from_output_format(output_format),
+            pause_frame_processing=True,
+            sample_rate=sample_rate,
             **kwargs,
         )
-        WebsocketService.__init__(self)
 
         self._api_key = api_key
         self._url = url
         self._settings = {
-            "sample_rate": sample_rate_from_output_format(output_format),
             "language": self.language_to_service_language(params.language)
             if params.language
             else None,
-            "output_format": output_format,
             "optimize_streaming_latency": params.optimize_streaming_latency,
             "stability": params.stability,
             "similarity_boost": params.similarity_boost,
             "style": params.style,
             "use_speaker_boost": params.use_speaker_boost,
+            "speed": params.speed,
             "auto_mode": str(params.auto_mode).lower(),
         }
         self.set_model_name(model)
         self.set_voice(voice_id)
+        self._output_format = ""  # initialized in start()
         self._voice_settings = self._set_voice_settings()
 
         # Indicates if we have sent TTSStartedFrame. It will reset to False when
@@ -218,35 +256,17 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
         self._started = False
         self._cumulative_time = 0
 
+        self._receive_task = None
+        self._keepalive_task = None
+
     def can_generate_metrics(self) -> bool:
         return True
 
-    def language_to_service_language(self, language: Language) -> str | None:
+    def language_to_service_language(self, language: Language) -> Optional[str]:
         return language_to_elevenlabs_language(language)
 
     def _set_voice_settings(self):
-        voice_settings = {}
-        if (
-            self._settings["stability"] is not None
-            and self._settings["similarity_boost"] is not None
-        ):
-            voice_settings["stability"] = self._settings["stability"]
-            voice_settings["similarity_boost"] = self._settings["similarity_boost"]
-            if self._settings["style"] is not None:
-                voice_settings["style"] = self._settings["style"]
-            if self._settings["use_speaker_boost"] is not None:
-                voice_settings["use_speaker_boost"] = self._settings["use_speaker_boost"]
-        else:
-            if self._settings["style"] is not None:
-                logger.warning(
-                    "'style' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
-                )
-            if self._settings["use_speaker_boost"] is not None:
-                logger.warning(
-                    "'use_speaker_boost' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
-                )
-
-        return voice_settings or None
+        return build_elevenlabs_voice_settings(self._settings)
 
     async def set_model(self, model: str):
         await super().set_model(model)
@@ -254,7 +274,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
         await self._disconnect()
         await self._connect()
 
-    async def _update_settings(self, settings: Dict[str, Any]):
+    async def _update_settings(self, settings: Mapping[str, Any]):
         prev_voice = self._voice_id
         await super()._update_settings(settings)
         if not prev_voice == self._voice_id:
@@ -264,6 +284,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+        self._output_format = output_format_from_sample_rate(self.sample_rate)
         await self._connect()
 
     async def stop(self, frame: EndFrame):
@@ -286,24 +307,14 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
             if isinstance(frame, TTSStoppedFrame):
                 await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)])
 
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        # If we received a TTSSpeakFrame and the LLM response included text (it
-        # might be that it's only a function calling response) we pause
-        # processing more frames until we receive a BotStoppedSpeakingFrame.
-        if isinstance(frame, TTSSpeakFrame):
-            await self.pause_processing_frames()
-        elif isinstance(frame, LLMFullResponseEndFrame) and self._started:
-            await self.pause_processing_frames()
-        elif isinstance(frame, BotStoppedSpeakingFrame):
-            await self.resume_processing_frames()
-
     async def _connect(self):
         await self._connect_websocket()
 
-        self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
-        self._keepalive_task = self.create_task(self._keepalive_task_handler())
+        if not self._receive_task:
+            self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
+
+        if not self._keepalive_task:
+            self._keepalive_task = self.create_task(self._keepalive_task_handler())
 
     async def _disconnect(self):
         if self._receive_task:
@@ -318,11 +329,14 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
 
     async def _connect_websocket(self):
         try:
+            if self._websocket:
+                return
+
             logger.debug("Connecting to ElevenLabs")
 
             voice_id = self._voice_id
             model = self.model_name
-            output_format = self._settings["output_format"]
+            output_format = self._output_format
             url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}"
 
             if self._settings["optimize_streaming_latency"]:
@@ -352,6 +366,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
         except Exception as e:
             logger.error(f"{self} initialization error: {e}")
             self._websocket = None
+            await self._call_event_handler("on_connection_error", f"{e}")
 
     async def _disconnect_websocket(self):
         try:
@@ -367,15 +382,20 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
         except Exception as e:
             logger.error(f"{self} error closing websocket: {e}")
 
+    def _get_websocket(self):
+        if self._websocket:
+            return self._websocket
+        raise Exception("Websocket not connected")
+
     async def _receive_messages(self):
-        async for message in self._websocket:
+        async for message in self._get_websocket():
             msg = json.loads(message)
             if msg.get("audio"):
                 await self.stop_ttfb_metrics()
                 self.start_word_timestamps()
 
                 audio = base64.b64decode(msg["audio"])
-                frame = TTSAudioRawFrame(audio, self._settings["sample_rate"], 1)
+                frame = TTSAudioRawFrame(audio, self.sample_rate, 1)
                 await self.push_frame(frame)
             if msg.get("alignment"):
                 word_times = calculate_word_times(msg["alignment"], self._cumulative_time)
@@ -385,7 +405,11 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
     async def _keepalive_task_handler(self):
         while True:
             await asyncio.sleep(10)
-            await self._send_text("")
+            try:
+                await self._send_text("")
+            except websockets.ConnectionClosed as e:
+                logger.warning(f"{self} keepalive error: {e}")
+                break
 
     async def _send_text(self, text: str):
         if self._websocket:
@@ -393,7 +417,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
             await self._websocket.send(json.dumps(msg))
 
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
             if not self._websocket:
@@ -428,7 +452,7 @@ class ElevenLabsHttpTTSService(TTSService):
         aiohttp_session: aiohttp ClientSession
         model: Model ID (default: "eleven_flash_v2_5" for low latency)
         base_url: API base URL
-        output_format: Audio output format (PCM)
+        sample_rate: Output sample rate
         params: Additional parameters for voice configuration
     """
 
@@ -439,6 +463,7 @@ class ElevenLabsHttpTTSService(TTSService):
         similarity_boost: Optional[float] = None
         style: Optional[float] = None
         use_speaker_boost: Optional[bool] = None
+        speed: Optional[float] = None
 
     def __init__(
         self,
@@ -448,65 +473,42 @@ class ElevenLabsHttpTTSService(TTSService):
         aiohttp_session: aiohttp.ClientSession,
         model: str = "eleven_flash_v2_5",
         base_url: str = "https://api.elevenlabs.io",
-        output_format: ElevenLabsOutputFormat = "pcm_24000",
+        sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
     ):
-        super().__init__(sample_rate=sample_rate_from_output_format(output_format), **kwargs)
+        super().__init__(sample_rate=sample_rate, **kwargs)
 
         self._api_key = api_key
         self._base_url = base_url
-        self._output_format = output_format
         self._params = params
         self._session = aiohttp_session
 
         self._settings = {
-            "sample_rate": sample_rate_from_output_format(output_format),
             "language": self.language_to_service_language(params.language)
             if params.language
             else None,
-            "output_format": output_format,
             "optimize_streaming_latency": params.optimize_streaming_latency,
             "stability": params.stability,
             "similarity_boost": params.similarity_boost,
             "style": params.style,
             "use_speaker_boost": params.use_speaker_boost,
+            "speed": params.speed,
         }
         self.set_model_name(model)
         self.set_voice(voice_id)
+        self._output_format = ""  # initialized in start()
         self._voice_settings = self._set_voice_settings()
 
     def can_generate_metrics(self) -> bool:
         return True
 
-    def _set_voice_settings(self) -> Optional[Dict[str, Union[float, bool]]]:
-        """Configure voice settings if stability and similarity_boost are provided.
+    def _set_voice_settings(self):
+        return build_elevenlabs_voice_settings(self._settings)
 
-        Returns:
-            Dictionary of voice settings or None if required parameters are missing.
-        """
-        voice_settings: Dict[str, Union[float, bool]] = {}
-        if (
-            self._settings["stability"] is not None
-            and self._settings["similarity_boost"] is not None
-        ):
-            voice_settings["stability"] = float(self._settings["stability"])
-            voice_settings["similarity_boost"] = float(self._settings["similarity_boost"])
-            if self._settings["style"] is not None:
-                voice_settings["style"] = float(self._settings["style"])
-            if self._settings["use_speaker_boost"] is not None:
-                voice_settings["use_speaker_boost"] = bool(self._settings["use_speaker_boost"])
-        else:
-            if self._settings["style"] is not None:
-                logger.warning(
-                    "'style' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
-                )
-            if self._settings["use_speaker_boost"] is not None:
-                logger.warning(
-                    "'use_speaker_boost' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
-                )
-
-        return voice_settings or None
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+        self._output_format = output_format_from_sample_rate(self.sample_rate)
 
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
         """Generate speech from text using ElevenLabs streaming API.
@@ -517,7 +519,7 @@ class ElevenLabsHttpTTSService(TTSService):
         Yields:
             Frames containing audio data and status information
         """
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         url = f"{self._base_url}/v1/text-to-speech/{self._voice_id}/stream"
 
@@ -565,18 +567,18 @@ class ElevenLabsHttpTTSService(TTSService):
                     return
 
                 await self.start_tts_usage_metrics(text)
+
+                # Process the streaming response
+                CHUNK_SIZE = 1024
+
                 yield TTSStartedFrame()
-
-                async for chunk in response.content:
-                    if chunk:
+                async for chunk in response.content.iter_chunked(CHUNK_SIZE):
+                    if len(chunk) > 0:
                         await self.stop_ttfb_metrics()
-                        yield TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
-
-                yield TTSStoppedFrame()
-
+                        yield TTSAudioRawFrame(chunk, self.sample_rate, 1)
         except Exception as e:
             logger.error(f"Error in run_tts: {e}")
             yield ErrorFrame(error=str(e))
-
         finally:
+            await self.stop_ttfb_metrics()
             yield TTSStoppedFrame()

src/pipecat/services/fal.py

@@ -4,8 +4,10 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+import asyncio
 import io
 import os
+import wave
 from typing import AsyncGenerator, Dict, Optional, Union
 
 import aiohttp
@@ -13,8 +15,10 @@ from loguru import logger
 from PIL import Image
 from pydantic import BaseModel
 
-from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
-from pipecat.services.ai_services import ImageGenService
+from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame, URLImageRawFrame
+from pipecat.services.ai_services import ImageGenService, SegmentedSTTService
+from pipecat.transcriptions.language import Language
+from pipecat.utils.time import time_now_iso8601
 
 try:
     import fal_client
@@ -26,6 +30,120 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+def language_to_fal_language(language: Language) -> Optional[str]:
+    """Language support for Fal's Wizper API."""
+    BASE_LANGUAGES = {
+        Language.AF: "af",
+        Language.AM: "am",
+        Language.AR: "ar",
+        Language.AS: "as",
+        Language.AZ: "az",
+        Language.BA: "ba",
+        Language.BE: "be",
+        Language.BG: "bg",
+        Language.BN: "bn",
+        Language.BO: "bo",
+        Language.BR: "br",
+        Language.BS: "bs",
+        Language.CA: "ca",
+        Language.CS: "cs",
+        Language.CY: "cy",
+        Language.DA: "da",
+        Language.DE: "de",
+        Language.EL: "el",
+        Language.EN: "en",
+        Language.ES: "es",
+        Language.ET: "et",
+        Language.EU: "eu",
+        Language.FA: "fa",
+        Language.FI: "fi",
+        Language.FO: "fo",
+        Language.FR: "fr",
+        Language.GL: "gl",
+        Language.GU: "gu",
+        Language.HA: "ha",
+        Language.HE: "he",
+        Language.HI: "hi",
+        Language.HR: "hr",
+        Language.HT: "ht",
+        Language.HU: "hu",
+        Language.HY: "hy",
+        Language.ID: "id",
+        Language.IS: "is",
+        Language.IT: "it",
+        Language.JA: "ja",
+        Language.JW: "jw",
+        Language.KA: "ka",
+        Language.KK: "kk",
+        Language.KM: "km",
+        Language.KN: "kn",
+        Language.KO: "ko",
+        Language.LA: "la",
+        Language.LB: "lb",
+        Language.LN: "ln",
+        Language.LO: "lo",
+        Language.LT: "lt",
+        Language.LV: "lv",
+        Language.MG: "mg",
+        Language.MI: "mi",
+        Language.MK: "mk",
+        Language.ML: "ml",
+        Language.MN: "mn",
+        Language.MR: "mr",
+        Language.MS: "ms",
+        Language.MT: "mt",
+        Language.MY: "my",
+        Language.NE: "ne",
+        Language.NL: "nl",
+        Language.NN: "nn",
+        Language.NO: "no",
+        Language.OC: "oc",
+        Language.PA: "pa",
+        Language.PL: "pl",
+        Language.PS: "ps",
+        Language.PT: "pt",
+        Language.RO: "ro",
+        Language.RU: "ru",
+        Language.SA: "sa",
+        Language.SD: "sd",
+        Language.SI: "si",
+        Language.SK: "sk",
+        Language.SL: "sl",
+        Language.SN: "sn",
+        Language.SO: "so",
+        Language.SQ: "sq",
+        Language.SR: "sr",
+        Language.SU: "su",
+        Language.SV: "sv",
+        Language.SW: "sw",
+        Language.TA: "ta",
+        Language.TE: "te",
+        Language.TG: "tg",
+        Language.TH: "th",
+        Language.TK: "tk",
+        Language.TL: "tl",
+        Language.TR: "tr",
+        Language.TT: "tt",
+        Language.UK: "uk",
+        Language.UR: "ur",
+        Language.UZ: "uz",
+        Language.VI: "vi",
+        Language.YI: "yi",
+        Language.YO: "yo",
+        Language.ZH: "zh",
+    }
+
+    result = BASE_LANGUAGES.get(language)
+
+    # If not found in base languages, try to find the base language from a variant
+    if not result:
+        lang_str = str(language.value)
+        base_code = lang_str.split("-")[0].lower()
+        result = base_code if base_code in BASE_LANGUAGES.values() else None
+
+    return result
+
+
 class FalImageGenService(ImageGenService):
     class InputParams(BaseModel):
         seed: Optional[int] = None
@@ -42,7 +160,7 @@ class FalImageGenService(ImageGenService):
         params: InputParams,
         aiohttp_session: aiohttp.ClientSession,
         model: str = "fal-ai/fast-sdxl",
-        key: str | None = None,
+        key: Optional[str] = None,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -53,6 +171,11 @@ class FalImageGenService(ImageGenService):
             os.environ["FAL_KEY"] = key
 
     async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
+        def load_image_bytes(encoded_image: bytes):
+            buffer = io.BytesIO(encoded_image)
+            image = Image.open(buffer)
+            return (image.tobytes(), image.size, image.format)
+
         logger.debug(f"Generating image from prompt: {prompt}")
 
         response = await fal_client.run_async(
@@ -73,10 +196,114 @@ class FalImageGenService(ImageGenService):
         logger.debug(f"Downloading image {image_url} ...")
         async with self._aiohttp_session.get(image_url) as response:
             logger.debug(f"Downloaded image {image_url}")
-            image_stream = io.BytesIO(await response.content.read())
-            image = Image.open(image_stream)
+            encoded_image = await response.content.read()
+            (image_bytes, size, format) = await asyncio.to_thread(load_image_bytes, encoded_image)
 
-            frame = URLImageRawFrame(
-                url=image_url, image=image.tobytes(), size=image.size, format=image.format
-            )
+            frame = URLImageRawFrame(url=image_url, image=image_bytes, size=size, format=format)
             yield frame
+
+
+class FalSTTService(SegmentedSTTService):
+    """Speech-to-text service using Fal's Wizper API.
+
+    This service uses Fal's Wizper API to perform speech-to-text transcription on audio
+    segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection.
+
+    Args:
+        api_key: Fal API key. If not provided, will check FAL_KEY environment variable.
+        sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate.
+        params: Configuration parameters for the Wizper API.
+        **kwargs: Additional arguments passed to SegmentedSTTService.
+    """
+
+    class InputParams(BaseModel):
+        """Configuration parameters for Fal's Wizper API.
+
+        Attributes:
+            language: Language of the audio input. Defaults to English.
+            task: Task to perform ('transcribe' or 'translate'). Defaults to 'transcribe'.
+            chunk_level: Level of chunking ('segment'). Defaults to 'segment'.
+            version: Version of Wizper model to use. Defaults to '3'.
+        """
+
+        language: Optional[Language] = Language.EN
+        task: str = "transcribe"
+        chunk_level: str = "segment"
+        version: str = "3"
+
+    def __init__(
+        self,
+        *,
+        api_key: Optional[str] = None,
+        sample_rate: Optional[int] = None,
+        params: InputParams = InputParams(),
+        **kwargs,
+    ):
+        super().__init__(
+            sample_rate=sample_rate,
+            **kwargs,
+        )
+
+        if api_key:
+            os.environ["FAL_KEY"] = api_key
+        elif "FAL_KEY" not in os.environ:
+            raise ValueError(
+                "FAL_KEY must be provided either through api_key parameter or environment variable"
+            )
+
+        self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY"))
+        self._settings = {
+            "task": params.task,
+            "language": self.language_to_service_language(params.language)
+            if params.language
+            else "en",
+            "chunk_level": params.chunk_level,
+            "version": params.version,
+        }
+
+    def can_generate_metrics(self) -> bool:
+        return True
+
+    def language_to_service_language(self, language: Language) -> Optional[str]:
+        return language_to_fal_language(language)
+
+    async def set_language(self, language: Language):
+        logger.info(f"Switching STT language to: [{language}]")
+        self._settings["language"] = self.language_to_service_language(language)
+
+    async def set_model(self, model: str):
+        await super().set_model(model)
+        logger.info(f"Switching STT model to: [{model}]")
+
+    async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
+        """Transcribes an audio segment using Fal's Wizper API.
+
+        Args:
+            audio: Raw audio bytes in WAV format (already converted by base class).
+
+        Yields:
+            Frame: TranscriptionFrame containing the transcribed text.
+
+        Note:
+            The audio is already in WAV format from the SegmentedSTTService.
+            Only non-empty transcriptions are yielded.
+        """
+        try:
+            # Send to Fal directly (audio is already in WAV format from base class)
+            data_uri = fal_client.encode(audio, "audio/x-wav")
+            response = await self._fal_client.run(
+                "fal-ai/wizper",
+                arguments={"audio_url": data_uri, **self._settings},
+            )
+
+            if response and "text" in response:
+                text = response["text"].strip()
+                if text:  # Only yield non-empty text
+                    logger.debug(f"Transcription: [{text}]")
+                    yield TranscriptionFrame(
+                        text, "", time_now_iso8601(), Language(self._settings["language"])
+                    )
+
+        except Exception as e:
+            logger.error(f"Fal Wizper error: {e}")
+            yield ErrorFrame(f"Fal Wizper error: {str(e)}")

src/pipecat/services/fireworks.py

@@ -8,19 +8,11 @@
 from typing import List
 
 from loguru import logger
+from openai.types.chat import ChatCompletionMessageParam
 
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai import OpenAILLMService
 
-try:
-    from openai.types.chat import ChatCompletionMessageParam
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error(
-        "In order to use Fireworks, you need to `pip install pipecat-ai[fireworks]`. Also, set `FIREWORKS_API_KEY` environment variable."
-    )
-    raise Exception(f"Missing module: {e}")
-
 
 class FireworksLLMService(OpenAILLMService):
     """A service for interacting with Fireworks AI using the OpenAI-compatible interface.

src/pipecat/services/fish.py

@@ -11,22 +11,18 @@ from loguru import logger
 from pydantic import BaseModel
 
 from pipecat.frames.frames import (
-    BotStoppedSpeakingFrame,
     CancelFrame,
     EndFrame,
     ErrorFrame,
     Frame,
-    LLMFullResponseEndFrame,
     StartFrame,
     StartInterruptionFrame,
     TTSAudioRawFrame,
-    TTSSpeakFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.ai_services import TTSService
-from pipecat.services.websocket_service import WebsocketService
+from pipecat.services.ai_services import InterruptibleTTSService
 from pipecat.transcriptions.language import Language
 
 try:
@@ -43,7 +39,7 @@ except ModuleNotFoundError as e:
 FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"]
 
 
-class FishAudioTTSService(TTSService, WebsocketService):
+class FishAudioTTSService(InterruptibleTTSService):
     class InputParams(BaseModel):
         language: Optional[Language] = Language.EN
         latency: Optional[str] = "normal"  # "normal" or "balanced"
@@ -56,11 +52,16 @@ class FishAudioTTSService(TTSService, WebsocketService):
         api_key: str,
         model: str,  # This is the reference_id
         output_format: FishAudioOutputFormat = "pcm",
-        sample_rate: int = 24000,
+        sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
     ):
-        super().__init__(sample_rate=sample_rate, **kwargs)
+        super().__init__(
+            push_stop_frames=True,
+            pause_frame_processing=True,
+            sample_rate=sample_rate,
+            **kwargs,
+        )
 
         self._api_key = api_key
         self._base_url = "wss://api.fish.audio/v1/tts/live"
@@ -70,7 +71,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
         self._started = False
 
         self._settings = {
-            "sample_rate": sample_rate,
+            "sample_rate": 0,
             "latency": params.latency,
             "format": output_format,
             "prosody": {
@@ -92,6 +93,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+        self._settings["sample_rate"] = self.sample_rate
         await self._connect()
 
     async def stop(self, frame: EndFrame):
@@ -104,16 +106,21 @@ class FishAudioTTSService(TTSService, WebsocketService):
 
     async def _connect(self):
         await self._connect_websocket()
-        self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
+        if not self._receive_task:
+            self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
 
     async def _disconnect(self):
-        await self._disconnect_websocket()
         if self._receive_task:
             await self.cancel_task(self._receive_task)
             self._receive_task = None
 
+        await self._disconnect_websocket()
+
     async def _connect_websocket(self):
         try:
+            if self._websocket:
+                return
+
             logger.debug("Connecting to Fish Audio")
             headers = {"Authorization": f"Bearer {self._api_key}"}
             self._websocket = await websockets.connect(self._base_url, extra_headers=headers)
@@ -125,6 +132,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
         except Exception as e:
             logger.error(f"Fish Audio initialization error: {e}")
             self._websocket = None
+            await self._call_event_handler("on_connection_error", f"{e}")
 
     async def _disconnect_websocket(self):
         try:
@@ -141,11 +149,24 @@ class FishAudioTTSService(TTSService, WebsocketService):
         except Exception as e:
             logger.error(f"Error closing websocket: {e}")
 
+    async def flush_audio(self):
+        """Flush any buffered audio by sending a flush event to Fish Audio."""
+        logger.trace(f"{self}: Flushing audio buffers")
+        if not self._websocket:
+            return
+        flush_message = {"event": "flush"}
+        await self._get_websocket().send(ormsgpack.packb(flush_message))
+
     def _get_websocket(self):
         if self._websocket:
             return self._websocket
         raise Exception("Websocket not connected")
 
+    async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
+        await super()._handle_interruption(frame, direction)
+        await self.stop_all_metrics()
+        self._request_id = None
+
     async def _receive_messages(self):
         async for message in self._get_websocket():
             try:
@@ -157,9 +178,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
                             audio_data = msg.get("audio")
                             # Only process larger chunks to remove msgpack overhead
                             if audio_data and len(audio_data) > 1024:
-                                frame = TTSAudioRawFrame(
-                                    audio_data, self._settings["sample_rate"], 1
-                                )
+                                frame = TTSAudioRawFrame(audio_data, self.sample_rate, 1)
                                 await self.push_frame(frame)
                                 await self.stop_ttfb_metrics()
                                 continue
@@ -167,23 +186,8 @@ class FishAudioTTSService(TTSService, WebsocketService):
             except Exception as e:
                 logger.error(f"Error processing message: {e}")
 
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if isinstance(frame, TTSSpeakFrame):
-            await self.pause_processing_frames()
-        elif isinstance(frame, LLMFullResponseEndFrame) and self._request_id:
-            await self.pause_processing_frames()
-        elif isinstance(frame, BotStoppedSpeakingFrame):
-            await self.resume_processing_frames()
-
-    async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
-        await super()._handle_interruption(frame, direction)
-        await self.stop_all_metrics()
-        self._request_id = None
-
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating Fish TTS: [{text}]")
+        logger.debug(f"{self}: Generating Fish TTS: [{text}]")
         try:
             if not self._websocket or self._websocket.closed:
                 await self._connect()

src/pipecat/services/gemini_multimodal_live/events.py

@@ -48,7 +48,7 @@ class AudioInputMessage(BaseModel):
     realtimeInput: RealtimeInput
 
     @classmethod
-    def from_raw_audio(cls, raw_audio: bytes, sample_rate=16000) -> "AudioInputMessage":
+    def from_raw_audio(cls, raw_audio: bytes, sample_rate: int) -> "AudioInputMessage":
         data = base64.b64encode(raw_audio).decode("utf-8")
         return cls(
             realtimeInput=RealtimeInput(

src/pipecat/services/gemini_multimodal_live/gemini.py

@@ -9,12 +9,14 @@ import base64
 import json
 from dataclasses import dataclass
 from enum import Enum
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Mapping, Optional, Union
 
 import websockets
 from loguru import logger
 from pydantic import BaseModel, Field
 
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter
 from pipecat.frames.frames import (
     BotStartedSpeakingFrame,
     BotStoppedSpeakingFrame,
@@ -36,6 +38,8 @@ from pipecat.frames.frames import (
     TTSAudioRawFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
+    TTSTextFrame,
+    UserImageRawFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
 )
@@ -115,10 +119,10 @@ class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator):
 
 
 class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggregator):
-    async def _push_aggregation(self):
-        # We don't want to store any images in the context. Revisit this later when the API evolves.
-        self._pending_image_frame_message = None
-        await super()._push_aggregation()
+    async def handle_user_image_frame(self, frame: UserImageRawFrame):
+        # We don't want to store any images in the context. Revisit this later
+        # when the API evolves.
+        pass
 
 
 @dataclass
@@ -152,6 +156,9 @@ class InputParams(BaseModel):
 
 
 class GeminiMultimodalLiveLLMService(LLMService):
+    # Overriding the default adapter to use the Gemini one.
+    adapter_class = GeminiLLMAdapter
+
     def __init__(
         self,
         *,
@@ -162,7 +169,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
         start_audio_paused: bool = False,
         start_video_paused: bool = False,
         system_instruction: Optional[str] = None,
-        tools: Optional[List[dict]] = None,
+        tools: Optional[Union[List[dict], ToolsSchema]] = None,
         transcribe_user_audio: bool = False,
         transcribe_model_audio: bool = False,
         params: InputParams = InputParams(),
@@ -203,6 +210,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
         self._bot_audio_buffer = bytearray()
         self._bot_text_buffer = ""
 
+        self._sample_rate = 24000
+
         self._settings = {
             "frequency_penalty": params.frequency_penalty,
             "max_tokens": params.max_tokens,
@@ -305,6 +314,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
         # context.add_message({"role": "assistant", "content": [{"type": "text", "text": text}]})
         await self.push_frame(LLMFullResponseStartFrame())
         await self.push_frame(LLMTextFrame(text=text))
+        await self.push_frame(TTSTextFrame(text=text))
         await self.push_frame(LLMFullResponseEndFrame())
 
     async def _transcribe_audio(self, audio, context):
@@ -332,10 +342,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
 
-        # logger.debug(f"Processing frame: {frame}")
-
         if isinstance(frame, TranscriptionFrame):
-            pass
+            await self.push_frame(frame, direction)
         elif isinstance(frame, OpenAILLMContextFrame):
             context: GeminiMultimodalLiveContext = GeminiMultimodalLiveContext.upgrade(
                 frame.context
@@ -352,31 +360,35 @@ class GeminiMultimodalLiveLLMService(LLMService):
                 # Support just one tool call per context frame for now
                 tool_result_message = context.messages[-1]
                 await self._tool_result(tool_result_message)
-
         elif isinstance(frame, InputAudioRawFrame):
             await self._send_user_audio(frame)
+            await self.push_frame(frame, direction)
         elif isinstance(frame, InputImageRawFrame):
             await self._send_user_video(frame)
+            await self.push_frame(frame, direction)
         elif isinstance(frame, StartInterruptionFrame):
             await self._handle_interruption()
+            await self.push_frame(frame, direction)
         elif isinstance(frame, UserStartedSpeakingFrame):
             await self._handle_user_started_speaking(frame)
+            await self.push_frame(frame, direction)
         elif isinstance(frame, UserStoppedSpeakingFrame):
             await self._handle_user_stopped_speaking(frame)
+            await self.push_frame(frame, direction)
         elif isinstance(frame, BotStartedSpeakingFrame):
             # Ignore this frame. Use the serverContent API message instead
-            pass
+            await self.push_frame(frame, direction)
         elif isinstance(frame, BotStoppedSpeakingFrame):
             # ignore this frame. Use the serverContent.turnComplete API message
-            pass
+            await self.push_frame(frame, direction)
         elif isinstance(frame, LLMMessagesAppendFrame):
             await self._create_single_response(frame.messages)
         elif isinstance(frame, LLMUpdateSettingsFrame):
             await self._update_settings(frame.settings)
         elif isinstance(frame, LLMSetToolsFrame):
             await self._update_settings()
-
-        await self.push_frame(frame, direction)
+        else:
+            await self.push_frame(frame, direction)
 
     #
     # websocket communication
@@ -433,7 +445,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
                 )
             if self._tools:
                 logger.debug(f"Gemini is configuring to use tools{self._tools}")
-                config.setup.tools = self._tools
+                config.setup.tools = self.get_llm_adapter().from_standard_tools(self._tools)
             await self.send_client_event(config)
 
         except Exception as e:
@@ -521,7 +533,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
         if self._audio_input_paused:
             return
         # Send all audio to Gemini
-        evt = events.AudioInputMessage.from_raw_audio(frame.audio)
+        evt = events.AudioInputMessage.from_raw_audio(frame.audio, frame.sample_rate)
         await self.send_client_event(evt)
         # Manage a buffer of audio to use for transcription
         audio = frame.audio
@@ -650,7 +662,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
         inline_data = part.inlineData
         if not inline_data:
             return
-        if inline_data.mimeType != "audio/pcm;rate=24000":
+        if inline_data.mimeType != f"audio/pcm;rate={self._sample_rate}":
             logger.warning(f"Unrecognized server_content format {inline_data.mimeType}")
             return
 
@@ -665,7 +677,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
         self._bot_audio_buffer.extend(audio)
         frame = TTSAudioRawFrame(
             audio=audio,
-            sample_rate=24000,
+            sample_rate=self._sample_rate,
             num_channels=1,
         )
         await self.push_frame(frame)
@@ -699,11 +711,39 @@ class GeminiMultimodalLiveLLMService(LLMService):
         await self.push_frame(TTSStoppedFrame())
 
     def create_context_aggregator(
-        self, context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = False
+        self,
+        context: OpenAILLMContext,
+        *,
+        user_kwargs: Mapping[str, Any] = {},
+        assistant_kwargs: Mapping[str, Any] = {},
     ) -> GeminiMultimodalLiveContextAggregatorPair:
+        """Create an instance of GeminiMultimodalLiveContextAggregatorPair from
+        an OpenAILLMContext. Constructor keyword arguments for both the user and
+        assistant aggregators can be provided.
+
+        Args:
+            context (OpenAILLMContext): The LLM context.
+            user_kwargs (Mapping[str, Any], optional): Additional keyword
+                arguments for the user context aggregator constructor. Defaults
+                to an empty mapping.
+            assistant_kwargs (Mapping[str, Any], optional): Additional keyword
+                arguments for the assistant context aggregator
+                constructor. Defaults to an empty mapping.
+
+        Returns:
+            GeminiMultimodalLiveContextAggregatorPair: A pair of context
+            aggregators, one for the user and one for the assistant,
+            encapsulated in an GeminiMultimodalLiveContextAggregatorPair.
+
+        """
+        context.set_llm_adapter(self.get_llm_adapter())
+
         GeminiMultimodalLiveContext.upgrade(context)
-        user = GeminiMultimodalLiveUserContextAggregator(context)
+        user = GeminiMultimodalLiveUserContextAggregator(context, **user_kwargs)
+
+        default_assistant_kwargs = {"expect_stripped_words": False}
+        default_assistant_kwargs.update(assistant_kwargs)
         assistant = GeminiMultimodalLiveAssistantContextAggregator(
-            user, expect_stripped_words=assistant_expect_stripped_words
+            context, **default_assistant_kwargs
         )
         return GeminiMultimodalLiveContextAggregatorPair(_user=user, _assistant=assistant)

src/pipecat/services/gladia.py

@@ -34,7 +34,7 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
-def language_to_gladia_language(language: Language) -> str | None:
+def language_to_gladia_language(language: Language) -> Optional[str]:
     BASE_LANGUAGES = {
         Language.AF: "af",
         Language.AM: "am",
@@ -131,12 +131,12 @@ def language_to_gladia_language(language: Language) -> str | None:
 
 class GladiaSTTService(STTService):
     class InputParams(BaseModel):
-        sample_rate: Optional[int] = 16000
         language: Optional[Language] = Language.EN
         endpointing: Optional[float] = 0.2
         maximum_duration_without_endpointing: Optional[int] = 10
         audio_enhancer: Optional[bool] = None
         words_accurate_timestamps: Optional[bool] = None
+        speech_threshold: Optional[float] = 0.99
 
     def __init__(
         self,
@@ -144,17 +144,17 @@ class GladiaSTTService(STTService):
         api_key: str,
         url: str = "https://api.gladia.io/v2/live",
         confidence: float = 0.5,
+        sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
     ):
-        super().__init__(**kwargs)
-
+        super().__init__(sample_rate=sample_rate, **kwargs)
         self._api_key = api_key
         self._url = url
         self._settings = {
             "encoding": "wav/pcm",
             "bit_depth": 16,
-            "sample_rate": params.sample_rate,
+            "sample_rate": 0,
             "channels": 1,
             "language_config": {
                 "languages": [self.language_to_service_language(params.language)]
@@ -166,32 +166,45 @@ class GladiaSTTService(STTService):
             "maximum_duration_without_endpointing": params.maximum_duration_without_endpointing,
             "pre_processing": {
                 "audio_enhancer": params.audio_enhancer,
+                "speech_threshold": params.speech_threshold,
             },
             "realtime_processing": {
                 "words_accurate_timestamps": params.words_accurate_timestamps,
             },
         }
         self._confidence = confidence
+        self._websocket = None
+        self._receive_task = None
 
-    def language_to_service_language(self, language: Language) -> str | None:
+    def language_to_service_language(self, language: Language) -> Optional[str]:
         return language_to_gladia_language(language)
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+        if self._websocket:
+            return
+        self._settings["sample_rate"] = self.sample_rate
         response = await self._setup_gladia()
         self._websocket = await websockets.connect(response["url"])
-        self._receive_task = self.create_task(self._receive_task_handler())
+        if not self._receive_task:
+            self._receive_task = self.create_task(self._receive_task_handler())
 
     async def stop(self, frame: EndFrame):
         await super().stop(frame)
         await self._send_stop_recording()
-        await self._websocket.close()
-        await self.wait_for_task(self._receive_task)
+        if self._websocket:
+            await self._websocket.close()
+            self._websocket = None
+        if self._receive_task:
+            await self.wait_for_task(self._receive_task)
+            self._receive_task = None
 
     async def cancel(self, frame: CancelFrame):
         await super().cancel(frame)
         await self._websocket.close()
-        await self.cancel_task(self._receive_task)
+        if self._receive_task:
+            await self.cancel_task(self._receive_task)
+            self._receive_task = None
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
         await self.start_processing_metrics()

src/pipecat/services/google/__init__.py

@@ -1,2 +1,3 @@
 from .frames import LLMSearchResponseFrame
 from .google import *
+from .rtvi import *

src/pipecat/services/google/rtvi.py

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from typing import List, Literal, Optional
+
+from pydantic import BaseModel
+
+from pipecat.frames.frames import Frame
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.processors.frameworks.rtvi import RTVIObserver
+from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame
+
+
+class RTVISearchResponseMessageData(BaseModel):
+    search_result: Optional[str]
+    rendered_content: Optional[str]
+    origins: List[LLMSearchOrigin]
+
+
+class RTVIBotLLMSearchResponseMessage(BaseModel):
+    label: Literal["rtvi-ai"] = "rtvi-ai"
+    type: Literal["bot-llm-search-response"] = "bot-llm-search-response"
+    data: RTVISearchResponseMessageData
+
+
+class GoogleRTVIObserver(RTVIObserver):
+    def __init__(self, rtvi: FrameProcessor):
+        super().__init__(rtvi)
+
+    async def on_push_frame(
+        self,
+        src: FrameProcessor,
+        dst: FrameProcessor,
+        frame: Frame,
+        direction: FrameDirection,
+        timestamp: int,
+    ):
+        await super().on_push_frame(src, dst, frame, direction, timestamp)
+
+        if isinstance(frame, LLMSearchResponseFrame):
+            await self._handle_llm_search_response_frame(frame)
+
+    async def _handle_llm_search_response_frame(self, frame: LLMSearchResponseFrame):
+        message = RTVIBotLLMSearchResponseMessage(
+            data=RTVISearchResponseMessageData(
+                search_result=frame.search_result,
+                origins=frame.origins,
+                rendered_content=frame.rendered_content,
+            )
+        )
+        await self.push_transport_message_urgent(message)

src/pipecat/services/grok.py

@@ -7,7 +7,7 @@
 
 import json
 from dataclasses import dataclass
-from typing import Optional
+from typing import Any, Mapping, Optional
 
 from loguru import logger
 
@@ -17,6 +17,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContext,
     OpenAILLMContextFrame,
 )
+from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.openai import (
     OpenAIAssistantContextAggregator,
     OpenAILLMService,
@@ -24,95 +25,15 @@ from pipecat.services.openai import (
 )
 
 
-class GrokAssistantContextAggregator(OpenAIAssistantContextAggregator):
-    """Custom assistant context aggregator for Grok that handles empty content requirement."""
-
-    async def _push_aggregation(self):
-        if not (
-            self._aggregation or self._function_call_result or self._pending_image_frame_message
-        ):
-            return
-
-        run_llm = False
-        properties: Optional[FunctionCallResultProperties] = None
-
-        aggregation = self._aggregation
-        self._reset()
-
-        try:
-            if self._function_call_result:
-                frame = self._function_call_result
-                properties = frame.properties
-                self._function_call_result = None
-                if frame.result:
-                    # Grok requires an empty content field for function calls
-                    self._context.add_message(
-                        {
-                            "role": "assistant",
-                            "content": "",  # Required by Grok
-                            "tool_calls": [
-                                {
-                                    "id": frame.tool_call_id,
-                                    "function": {
-                                        "name": frame.function_name,
-                                        "arguments": json.dumps(frame.arguments),
-                                    },
-                                    "type": "function",
-                                }
-                            ],
-                        }
-                    )
-                    self._context.add_message(
-                        {
-                            "role": "tool",
-                            "content": json.dumps(frame.result),
-                            "tool_call_id": frame.tool_call_id,
-                        }
-                    )
-                    if properties and properties.run_llm is not None:
-                        # If the tool call result has a run_llm property, use it
-                        run_llm = properties.run_llm
-                    else:
-                        # Default behavior is to run the LLM if there are no function calls in progress
-                        run_llm = not bool(self._function_calls_in_progress)
-
-            else:
-                self._context.add_message({"role": "assistant", "content": aggregation})
-
-            if self._pending_image_frame_message:
-                frame = self._pending_image_frame_message
-                self._pending_image_frame_message = None
-                self._context.add_image_frame_message(
-                    format=frame.user_image_raw_frame.format,
-                    size=frame.user_image_raw_frame.size,
-                    image=frame.user_image_raw_frame.image,
-                    text=frame.text,
-                )
-                run_llm = True
-
-            if run_llm:
-                await self._user_context_aggregator.push_context_frame()
-
-            # Emit the on_context_updated callback once the function call result is added to the context
-            if properties and properties.on_context_updated is not None:
-                await properties.on_context_updated()
-
-            frame = OpenAILLMContextFrame(self._context)
-            await self.push_frame(frame)
-
-        except Exception as e:
-            logger.error(f"Error processing frame: {e}")
-
-
 @dataclass
 class GrokContextAggregatorPair:
     _user: "OpenAIUserContextAggregator"
-    _assistant: "GrokAssistantContextAggregator"
+    _assistant: "OpenAIAssistantContextAggregator"
 
     def user(self) -> "OpenAIUserContextAggregator":
         return self._user
 
-    def assistant(self) -> "GrokAssistantContextAggregator":
+    def assistant(self) -> "OpenAIAssistantContextAggregator":
         return self._assistant
 
 
@@ -125,7 +46,7 @@ class GrokLLMService(OpenAILLMService):
     Args:
         api_key (str): The API key for accessing Grok's API
         base_url (str, optional): The base URL for Grok API. Defaults to "https://api.x.ai/v1"
-        model (str, optional): The model identifier to use. Defaults to "grok-beta"
+        model (str, optional): The model identifier to use. Defaults to "grok-2"
         **kwargs: Additional keyword arguments passed to OpenAILLMService
     """
 
@@ -134,7 +55,7 @@ class GrokLLMService(OpenAILLMService):
         *,
         api_key: str,
         base_url: str = "https://api.x.ai/v1",
-        model: str = "grok-beta",
+        model: str = "grok-2",
         **kwargs,
     ):
         super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
@@ -206,12 +127,34 @@ class GrokLLMService(OpenAILLMService):
         if tokens.completion_tokens > self._completion_tokens:
             self._completion_tokens = tokens.completion_tokens
 
-    @staticmethod
     def create_context_aggregator(
-        context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True
+        self,
+        context: OpenAILLMContext,
+        *,
+        user_kwargs: Mapping[str, Any] = {},
+        assistant_kwargs: Mapping[str, Any] = {},
     ) -> GrokContextAggregatorPair:
-        user = OpenAIUserContextAggregator(context)
-        assistant = GrokAssistantContextAggregator(
-            user, expect_stripped_words=assistant_expect_stripped_words
-        )
+        """Create an instance of GrokContextAggregatorPair from an
+        OpenAILLMContext. Constructor keyword arguments for both the user and
+        assistant aggregators can be provided.
+
+        Args:
+            context (OpenAILLMContext): The LLM context.
+            user_kwargs (Mapping[str, Any], optional): Additional keyword
+                arguments for the user context aggregator constructor. Defaults
+                to an empty mapping.
+            assistant_kwargs (Mapping[str, Any], optional): Additional keyword
+                arguments for the assistant context aggregator
+                constructor. Defaults to an empty mapping.
+
+        Returns:
+            GrokContextAggregatorPair: A pair of context aggregators, one for
+            the user and one for the assistant, encapsulated in an
+            GrokContextAggregatorPair.
+
+        """
+        context.set_llm_adapter(self.get_llm_adapter())
+
+        user = OpenAIUserContextAggregator(context, **user_kwargs)
+        assistant = OpenAIAssistantContextAggregator(context, **assistant_kwargs)
         return GrokContextAggregatorPair(_user=user, _assistant=assistant)

src/pipecat/services/groq.py

@@ -5,9 +5,25 @@
 #
 
 
-from loguru import logger
+from typing import AsyncGenerator, Optional
 
+from loguru import logger
+from pydantic import BaseModel
+
+from pipecat.frames.frames import Frame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame
+from pipecat.services.ai_services import TTSService
+from pipecat.services.base_whisper import BaseWhisperSTTService, Transcription
 from pipecat.services.openai import OpenAILLMService
+from pipecat.transcriptions.language import Language
+
+try:
+    from groq import AsyncGroq
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use Groq, you need to `pip install pipecat-ai[groq]`. Also, set a `GROQ_API_KEY` environment variable."
+    )
+    raise Exception(f"Missing module: {e}")
 
 
 class GroqLLMService(OpenAILLMService):
@@ -19,7 +35,7 @@ class GroqLLMService(OpenAILLMService):
     Args:
         api_key (str): The API key for accessing Groq's API
         base_url (str, optional): The base URL for Groq API. Defaults to "https://api.groq.com/openai/v1"
-        model (str, optional): The model identifier to use. Defaults to "llama-3.1-70b-versatile"
+        model (str, optional): The model identifier to use. Defaults to "llama-3.3-70b-versatile"
         **kwargs: Additional keyword arguments passed to OpenAILLMService
     """
 
@@ -28,7 +44,7 @@ class GroqLLMService(OpenAILLMService):
         *,
         api_key: str,
         base_url: str = "https://api.groq.com/openai/v1",
-        model: str = "llama-3.1-70b-versatile",
+        model: str = "llama-3.3-70b-versatile",
         **kwargs,
     ):
         super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
@@ -37,3 +53,125 @@ class GroqLLMService(OpenAILLMService):
         """Create OpenAI-compatible client for Groq API endpoint."""
         logger.debug(f"Creating Groq client with api {base_url}")
         return super().create_client(api_key, base_url, **kwargs)
+
+
+class GroqSTTService(BaseWhisperSTTService):
+    """Groq Whisper speech-to-text service.
+
+    Uses Groq's Whisper API to convert audio to text. Requires a Groq API key
+    set via the api_key parameter or GROQ_API_KEY environment variable.
+
+    Args:
+        model: Whisper model to use. Defaults to "whisper-large-v3-turbo".
+        api_key: Groq API key. Defaults to None.
+        base_url: API base URL. Defaults to "https://api.groq.com/openai/v1".
+        language: Language of the audio input. Defaults to English.
+        prompt: Optional text to guide the model's style or continue a previous segment.
+        temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0.
+        **kwargs: Additional arguments passed to BaseWhisperSTTService.
+    """
+
+    def __init__(
+        self,
+        *,
+        model: str = "whisper-large-v3-turbo",
+        api_key: Optional[str] = None,
+        base_url: str = "https://api.groq.com/openai/v1",
+        language: Optional[Language] = Language.EN,
+        prompt: Optional[str] = None,
+        temperature: Optional[float] = None,
+        **kwargs,
+    ):
+        super().__init__(
+            model=model,
+            api_key=api_key,
+            base_url=base_url,
+            language=language,
+            prompt=prompt,
+            temperature=temperature,
+            **kwargs,
+        )
+
+    async def _transcribe(self, audio: bytes) -> Transcription:
+        assert self._language is not None  # Assigned in the BaseWhisperSTTService class
+
+        # Build kwargs dict with only set parameters
+        kwargs = {
+            "file": ("audio.wav", audio, "audio/wav"),
+            "model": self.model_name,
+            "response_format": "json",
+            "language": self._language,
+        }
+
+        if self._prompt is not None:
+            kwargs["prompt"] = self._prompt
+
+        if self._temperature is not None:
+            kwargs["temperature"] = self._temperature
+
+        return await self._client.audio.transcriptions.create(**kwargs)
+
+
+class GroqTTSService(TTSService):
+    class InputParams(BaseModel):
+        language: Optional[Language] = Language.EN
+        speed: Optional[float] = 1.0
+        seed: Optional[int] = None
+
+    GROQ_SAMPLE_RATE = 48000  # Groq TTS only supports 48kHz sample rate
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        output_format: str = "wav",
+        params: InputParams = InputParams(),
+        model_name: str = "playai-tts",
+        voice_id: str = "Celeste-PlayAI",
+        sample_rate: Optional[int] = GROQ_SAMPLE_RATE,
+        **kwargs,
+    ):
+        if sample_rate != self.GROQ_SAMPLE_RATE:
+            logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ")
+        super().__init__(
+            pause_frame_processing=True,
+            sample_rate=sample_rate,
+            **kwargs,
+        )
+
+        self._api_key = api_key
+        self._model_name = model_name
+        self._output_format = output_format
+        self._voice_id = voice_id
+        self._params = params
+
+        self._client = AsyncGroq(api_key=self._api_key)
+
+    def can_generate_metrics(self) -> bool:
+        return True
+
+    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+        logger.debug(f"{self}: Generating TTS [{text}]")
+        measuring_ttfb = True
+        await self.start_ttfb_metrics()
+        yield TTSStartedFrame()
+
+        response = await self._client.audio.speech.create(
+            model=self._model_name,
+            voice=self._voice_id,
+            response_format=self._output_format,
+            input=text,
+        )
+
+        async for data in response.iter_bytes():
+            if measuring_ttfb:
+                await self.stop_ttfb_metrics()
+                measuring_ttfb = False
+            # remove wav header if present
+            if data.startswith(b"RIFF"):
+                data = data[44:]
+                if len(data) == 0:
+                    continue
+            yield TTSAudioRawFrame(data, self.sample_rate, 1)
+
+        yield TTSStoppedFrame()

src/pipecat/services/lmnt.py

@@ -5,7 +5,7 @@
 #
 
 import json
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 
 from loguru import logger
 
@@ -21,8 +21,7 @@ from pipecat.frames.frames import (
     TTSStoppedFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.ai_services import TTSService
-from pipecat.services.websocket_service import WebsocketService
+from pipecat.services.ai_services import InterruptibleTTSService
 from pipecat.transcriptions.language import Language
 
 # See .env.example for LMNT configuration needed
@@ -36,7 +35,7 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
-def language_to_lmnt_language(language: Language) -> str | None:
+def language_to_lmnt_language(language: Language) -> Optional[str]:
     BASE_LANGUAGES = {
         Language.DE: "de",
         Language.EN: "en",
@@ -60,37 +59,36 @@ def language_to_lmnt_language(language: Language) -> str | None:
     return result
 
 
-class LmntTTSService(TTSService, WebsocketService):
+class LmntTTSService(InterruptibleTTSService):
     def __init__(
         self,
         *,
         api_key: str,
         voice_id: str,
-        sample_rate: int = 24000,
+        sample_rate: Optional[int] = None,
         language: Language = Language.EN,
         **kwargs,
     ):
-        TTSService.__init__(
-            self,
+        super().__init__(
             push_stop_frames=True,
+            pause_frame_processing=True,
             sample_rate=sample_rate,
             **kwargs,
         )
-        WebsocketService.__init__(self)
 
         self._api_key = api_key
         self._voice_id = voice_id
         self._settings = {
-            "sample_rate": sample_rate,
             "language": self.language_to_service_language(language),
             "format": "raw",  # Use raw format for direct PCM data
         }
         self._started = False
+        self._receive_task = None
 
     def can_generate_metrics(self) -> bool:
         return True
 
-    def language_to_service_language(self, language: Language) -> str | None:
+    def language_to_service_language(self, language: Language) -> Optional[str]:
         return language_to_lmnt_language(language)
 
     async def start(self, frame: StartFrame):
@@ -113,18 +111,22 @@ class LmntTTSService(TTSService, WebsocketService):
     async def _connect(self):
         await self._connect_websocket()
 
-        self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
+        if not self._receive_task:
+            self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
 
     async def _disconnect(self):
-        await self._disconnect_websocket()
-
         if self._receive_task:
             await self.cancel_task(self._receive_task)
             self._receive_task = None
 
+        await self._disconnect_websocket()
+
     async def _connect_websocket(self):
         """Connect to LMNT websocket."""
         try:
+            if self._websocket:
+                return
+
             logger.debug("Connecting to LMNT")
 
             # Build initial connection message
@@ -132,7 +134,7 @@ class LmntTTSService(TTSService, WebsocketService):
                 "X-API-Key": self._api_key,
                 "voice": self._voice_id,
                 "format": self._settings["format"],
-                "sample_rate": self._settings["sample_rate"],
+                "sample_rate": self.sample_rate,
                 "language": self._settings["language"],
             }
 
@@ -145,6 +147,7 @@ class LmntTTSService(TTSService, WebsocketService):
         except Exception as e:
             logger.error(f"{self} initialization error: {e}")
             self._websocket = None
+            await self._call_event_handler("on_connection_error", f"{e}")
 
     async def _disconnect_websocket(self):
         """Disconnect from LMNT websocket."""
@@ -153,8 +156,9 @@ class LmntTTSService(TTSService, WebsocketService):
 
             if self._websocket:
                 logger.debug("Disconnecting from LMNT")
-                # Send EOF message before closing
-                await self._websocket.send(json.dumps({"eof": True}))
+                # NOTE(aleix): sending EOF message before closing is causing
+                # errors on the websocket, so we just skip it for now.
+                # await self._websocket.send(json.dumps({"eof": True}))
                 await self._websocket.close()
                 self._websocket = None
 
@@ -167,6 +171,11 @@ class LmntTTSService(TTSService, WebsocketService):
             return self._websocket
         raise Exception("Websocket not connected")
 
+    async def flush_audio(self):
+        if not self._websocket:
+            return
+        await self._get_websocket().send(json.dumps({"flush": True}))
+
     async def _receive_messages(self):
         """Receive messages from LMNT websocket."""
         async for message in self._get_websocket():
@@ -175,7 +184,7 @@ class LmntTTSService(TTSService, WebsocketService):
                 await self.stop_ttfb_metrics()
                 frame = TTSAudioRawFrame(
                     audio=message,
-                    sample_rate=self._settings["sample_rate"],
+                    sample_rate=self.sample_rate,
                     num_channels=1,
                 )
                 await self.push_frame(frame)
@@ -193,7 +202,7 @@ class LmntTTSService(TTSService, WebsocketService):
 
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
         """Generate TTS audio from text."""
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
             if not self._websocket:

src/pipecat/services/neuphonic.py

@@ -0,0 +1,345 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import base64
+import json
+from typing import Any, AsyncGenerator, Mapping, Optional
+
+from loguru import logger
+from pydantic import BaseModel
+
+from pipecat.frames.frames import (
+    BotStoppedSpeakingFrame,
+    CancelFrame,
+    EndFrame,
+    ErrorFrame,
+    Frame,
+    LLMFullResponseEndFrame,
+    StartFrame,
+    StartInterruptionFrame,
+    TTSAudioRawFrame,
+    TTSSpeakFrame,
+    TTSStartedFrame,
+    TTSStoppedFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.ai_services import InterruptibleTTSService, TTSService
+from pipecat.transcriptions.language import Language
+
+# See .env.example for Neuphonic configuration needed
+try:
+    import websockets
+    from pyneuphonic import Neuphonic, TTSConfig
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use Neuphonic, you need to `pip install pipecat-ai[neuphonic]`. Also, set `NEUPHONIC_API_KEY` environment variable."
+    )
+    raise Exception(f"Missing module: {e}")
+
+
+def language_to_neuphonic_lang_code(language: Language) -> Optional[str]:
+    BASE_LANGUAGES = {
+        Language.DE: "de",
+        Language.EN: "en",
+        Language.ES: "es",
+        Language.NL: "nl",
+        Language.AR: "ar",
+        Language.FR: "fr",
+        Language.PT: "pt",
+        Language.RU: "ru",
+        Language.HI: "HI",
+        Language.ZH: "zh",
+    }
+
+    result = BASE_LANGUAGES.get(language)
+
+    # If not found in base languages, try to find the base language from a variant
+    if not result:
+        # Convert enum value to string and get the base language part (e.g. es-ES -> es)
+        lang_str = str(language.value)
+        base_code = lang_str.split("-")[0].lower()
+        # Look up the base code in our supported languages
+        result = base_code if base_code in BASE_LANGUAGES.values() else None
+
+    return result
+
+
+class NeuphonicTTSService(InterruptibleTTSService):
+    class InputParams(BaseModel):
+        language: Optional[Language] = Language.EN
+        speed: Optional[float] = 1.0
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        voice_id: Optional[str] = None,
+        url: str = "wss://api.neuphonic.com",
+        sample_rate: Optional[int] = 22050,
+        encoding: str = "pcm_linear",
+        params: InputParams = InputParams(),
+        **kwargs,
+    ):
+        super().__init__(
+            aggregate_sentences=True,
+            push_text_frames=False,
+            push_stop_frames=True,
+            stop_frame_timeout_s=2.0,
+            sample_rate=sample_rate,
+            **kwargs,
+        )
+
+        self._api_key = api_key
+        self._url = url
+        self._settings = {
+            "lang_code": self.language_to_service_language(params.language),
+            "speed": params.speed,
+            "encoding": encoding,
+            "sampling_rate": sample_rate,
+        }
+        self.set_voice(voice_id)
+
+        # Indicates if we have sent TTSStartedFrame. It will reset to False when
+        # there's an interruption or TTSStoppedFrame.
+        self._started = False
+        self._cumulative_time = 0
+
+    def can_generate_metrics(self) -> bool:
+        return True
+
+    def language_to_service_language(self, language: Language) -> Optional[str]:
+        return language_to_neuphonic_lang_code(language)
+
+    async def _update_settings(self, settings: Mapping[str, Any]):
+        if "voice_id" in settings:
+            self.set_voice(settings["voice_id"])
+
+        await super()._update_settings(settings)
+        await self._disconnect()
+        await self._connect()
+        logger.info(f"Switching TTS to settings: [{self._settings}]")
+
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+        await self._connect()
+
+    async def stop(self, frame: EndFrame):
+        await super().stop(frame)
+        await self._disconnect()
+
+    async def cancel(self, frame: CancelFrame):
+        await super().cancel(frame)
+        await self._disconnect()
+
+    async def flush_audio(self):
+        if self._websocket:
+            msg = {"text": "<STOP>"}
+            await self._websocket.send(json.dumps(msg))
+
+    async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
+        await super().push_frame(frame, direction)
+        if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
+            self._started = False
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        # If we received a TTSSpeakFrame and the LLM response included text (it
+        # might be that it's only a function calling response) we pause
+        # processing more frames until we receive a BotStoppedSpeakingFrame.
+        if isinstance(frame, TTSSpeakFrame):
+            await self.pause_processing_frames()
+        elif isinstance(frame, LLMFullResponseEndFrame) and self._started:
+            await self.pause_processing_frames()
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            await self.resume_processing_frames()
+
+    async def _connect(self):
+        await self._connect_websocket()
+
+        self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
+        self._keepalive_task = self.create_task(self._keepalive_task_handler())
+
+    async def _disconnect(self):
+        if self._receive_task:
+            await self.cancel_task(self._receive_task)
+            self._receive_task = None
+
+        if self._keepalive_task:
+            await self.cancel_task(self._keepalive_task)
+            self._keepalive_task = None
+
+        await self._disconnect_websocket()
+
+    async def _connect_websocket(self):
+        try:
+            logger.debug("Connecting to Neuphonic")
+
+            tts_config = {
+                **self._settings,
+                "voice_id": self._voice_id,
+            }
+
+            query_params = [f"api_key={self._api_key}"]
+            for key, value in tts_config.items():
+                if value is not None:
+                    query_params.append(f"{key}={value}")
+
+            url = f"{self._url}/speak/{self._settings['lang_code']}?{'&'.join(query_params)}"
+
+            self._websocket = await websockets.connect(url)
+
+        except Exception as e:
+            logger.error(f"{self} initialization error: {e}")
+            self._websocket = None
+            await self._call_event_handler("on_connection_error", f"{e}")
+
+    async def _disconnect_websocket(self):
+        try:
+            await self.stop_all_metrics()
+
+            if self._websocket:
+                logger.debug("Disconnecting from Neuphonic")
+                await self._websocket.close()
+                self._websocket = None
+
+            self._started = False
+        except Exception as e:
+            logger.error(f"{self} error closing websocket: {e}")
+
+    async def _receive_messages(self):
+        async for message in self._websocket:
+            if isinstance(message, str):
+                msg = json.loads(message)
+                if msg.get("data", {}).get("audio") is not None:
+                    await self.stop_ttfb_metrics()
+
+                    audio = base64.b64decode(msg["data"]["audio"])
+                    frame = TTSAudioRawFrame(audio, self.sample_rate, 1)
+                    await self.push_frame(frame)
+
+    async def _keepalive_task_handler(self):
+        while True:
+            await asyncio.sleep(10)
+            await self._send_text("")
+
+    async def _send_text(self, text: str):
+        if self._websocket:
+            msg = {"text": text}
+            logger.debug(f"Sending text to websocket: {msg}")
+            await self._websocket.send(json.dumps(msg))
+
+    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+        logger.debug(f"Generating TTS: [{text}]")
+
+        try:
+            if not self._websocket:
+                await self._connect()
+
+            try:
+                if not self._started:
+                    await self.start_ttfb_metrics()
+                    yield TTSStartedFrame()
+                    self._started = True
+                    self._cumulative_time = 0
+
+                await self._send_text(text)
+                await self.start_tts_usage_metrics(text)
+            except Exception as e:
+                logger.error(f"{self} error sending message: {e}")
+                yield TTSStoppedFrame()
+                await self._disconnect()
+                await self._connect()
+                return
+            yield None
+        except Exception as e:
+            logger.error(f"{self} exception: {e}")
+
+
+class NeuphonicHttpTTSService(TTSService):
+    """Neuphonic Text-to-Speech service using HTTP streaming.
+
+    Args:
+        api_key: Neuphonic API key
+        voice_id: ID of the voice to use
+        url: Base URL for the Neuphonic API (default: "https://api.neuphonic.com")
+        sample_rate: Sample rate for audio output (default: 22050Hz)
+        encoding: Audio encoding format (default: "pcm_linear")
+        params: Additional parameters for TTS generation including language and speed
+        **kwargs: Additional keyword arguments passed to the parent class
+    """
+
+    class InputParams(BaseModel):
+        language: Optional[Language] = Language.EN
+        speed: Optional[float] = 1.0
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        voice_id: Optional[str] = None,
+        url: str = "https://api.neuphonic.com",
+        sample_rate: Optional[int] = 22050,
+        encoding: str = "pcm_linear",
+        params: InputParams = InputParams(),
+        **kwargs,
+    ):
+        super().__init__(sample_rate=sample_rate, **kwargs)
+
+        self._api_key = api_key
+        self._url = url
+        self._settings = {
+            "lang_code": self.language_to_service_language(params.language),
+            "speed": params.speed,
+            "encoding": encoding,
+            "sampling_rate": sample_rate,
+        }
+        self.set_voice(voice_id)
+
+    def can_generate_metrics(self) -> bool:
+        return True
+
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+
+    async def flush_audio(self):
+        pass
+
+    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+        """Generate speech from text using Neuphonic streaming API.
+
+        Args:
+            text: The text to convert to speech
+        Yields:
+            Frames containing audio data and status information
+        """
+        logger.debug(f"Generating TTS: [{text}]")
+
+        client = Neuphonic(api_key=self._api_key, base_url=self._url.replace("https://", ""))
+
+        sse = client.tts.AsyncSSEClient()
+
+        try:
+            await self.start_ttfb_metrics()
+            response = sse.send(text, TTSConfig(**self._settings, voice_id=self._voice_id))
+
+            await self.start_tts_usage_metrics(text)
+            yield TTSStartedFrame()
+
+            async for message in response:
+                if message.status_code != 200:
+                    logger.error(f"{self} error: {message.errors}")
+                    yield ErrorFrame(error=f"Neuphonic API error: {message.errors}")
+
+                await self.stop_ttfb_metrics()
+                yield TTSAudioRawFrame(message.data.audio, self.sample_rate, 1)
+        except Exception as e:
+            logger.error(f"Error in run_tts: {e}")
+            yield ErrorFrame(error=str(e))
+        finally:
+            yield TTSStoppedFrame()

src/pipecat/services/openai.py

@@ -8,33 +8,39 @@ import base64
 import io
 import json
 from dataclasses import dataclass
-from typing import Any, AsyncGenerator, Dict, List, Literal, Optional
+from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional
 
 import aiohttp
 import httpx
 from loguru import logger
+from openai import (
+    NOT_GIVEN,
+    AsyncOpenAI,
+    AsyncStream,
+    BadRequestError,
+    DefaultAsyncHttpxClient,
+)
+from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
 from PIL import Image
 from pydantic import BaseModel, Field
 
 from pipecat.frames.frames import (
     ErrorFrame,
     Frame,
+    FunctionCallCancelFrame,
     FunctionCallInProgressFrame,
     FunctionCallResultFrame,
-    FunctionCallResultProperties,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesFrame,
     LLMTextFrame,
     LLMUpdateSettingsFrame,
-    OpenAILLMContextAssistantTimestampFrame,
-    StartInterruptionFrame,
+    StartFrame,
     TTSAudioRawFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
     URLImageRawFrame,
     UserImageRawFrame,
-    UserImageRequestFrame,
     VisionImageRawFrame,
 )
 from pipecat.metrics.metrics import LLMTokenUsage
@@ -47,25 +53,13 @@ from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContextFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.ai_services import ImageGenService, LLMService, TTSService
-from pipecat.utils.time import time_now_iso8601
-
-try:
-    from openai import (
-        NOT_GIVEN,
-        AsyncOpenAI,
-        AsyncStream,
-        BadRequestError,
-        DefaultAsyncHttpxClient,
-    )
-    from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error(
-        "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable."
-    )
-    raise Exception(f"Missing module: {e}")
-
+from pipecat.services.ai_services import (
+    ImageGenService,
+    LLMService,
+    TTSService,
+)
+from pipecat.services.base_whisper import BaseWhisperSTTService, Transcription
+from pipecat.transcriptions.language import Language
 
 ValidVoice = Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"]
 
@@ -103,7 +97,7 @@ class BaseOpenAILLMService(LLMService):
         seed: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=0)
         temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=2.0)
         # Note: top_k is currently not supported by the OpenAI client library,
-        # so top_k is ignore right now.
+        # so top_k is ignored right now.
         top_k: Optional[int] = Field(default=None, ge=0)
         top_p: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
         max_tokens: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=1)
@@ -118,6 +112,7 @@ class BaseOpenAILLMService(LLMService):
         base_url=None,
         organization=None,
         project=None,
+        default_headers: Mapping[str, str] | None = None,
         params: InputParams = InputParams(),
         **kwargs,
     ):
@@ -134,10 +129,23 @@ class BaseOpenAILLMService(LLMService):
         }
         self.set_model_name(model)
         self._client = self.create_client(
-            api_key=api_key, base_url=base_url, organization=organization, project=project, **kwargs
+            api_key=api_key,
+            base_url=base_url,
+            organization=organization,
+            project=project,
+            default_headers=default_headers,
+            **kwargs,
         )
 
-    def create_client(self, api_key=None, base_url=None, organization=None, project=None, **kwargs):
+    def create_client(
+        self,
+        api_key=None,
+        base_url=None,
+        organization=None,
+        project=None,
+        default_headers=None,
+        **kwargs,
+    ):
         return AsyncOpenAI(
             api_key=api_key,
             base_url=base_url,
@@ -148,6 +156,7 @@ class BaseOpenAILLMService(LLMService):
                     max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None
                 )
             ),
+            default_headers=default_headers,
         )
 
     def can_generate_metrics(self) -> bool:
@@ -180,7 +189,7 @@ class BaseOpenAILLMService(LLMService):
     async def _stream_chat_completions(
         self, context: OpenAILLMContext
     ) -> AsyncStream[ChatCompletionChunk]:
-        logger.debug(f"Generating chat: {context.get_messages_for_logging()}")
+        logger.debug(f"{self}: Generating chat [{context.get_messages_for_logging()}]")
 
         messages: List[ChatCompletionMessageParam] = context.get_messages()
 
@@ -259,7 +268,6 @@ class BaseOpenAILLMService(LLMService):
                 if tool_call.function and tool_call.function.name:
                     function_name += tool_call.function.name
                     tool_call_id = tool_call.id
-                    await self.call_start_function(context, function_name)
                 if tool_call.function and tool_call.function.arguments:
                     # Keep iterating through the response to collect all the argument fragments
                     arguments += tool_call.function.arguments
@@ -313,11 +321,15 @@ class BaseOpenAILLMService(LLMService):
             await self.push_frame(frame, direction)
 
         if context:
-            await self.push_frame(LLMFullResponseStartFrame())
-            await self.start_processing_metrics()
-            await self._process_context(context)
-            await self.stop_processing_metrics()
-            await self.push_frame(LLMFullResponseEndFrame())
+            try:
+                await self.push_frame(LLMFullResponseStartFrame())
+                await self.start_processing_metrics()
+                await self._process_context(context)
+            except httpx.TimeoutException:
+                await self._call_event_handler("on_completion_timeout")
+            finally:
+                await self.stop_processing_metrics()
+                await self.push_frame(LLMFullResponseEndFrame())
 
 
 @dataclass
@@ -342,14 +354,35 @@ class OpenAILLMService(BaseOpenAILLMService):
     ):
         super().__init__(model=model, params=params, **kwargs)
 
-    @staticmethod
     def create_context_aggregator(
-        context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True
+        self,
+        context: OpenAILLMContext,
+        *,
+        user_kwargs: Mapping[str, Any] = {},
+        assistant_kwargs: Mapping[str, Any] = {},
     ) -> OpenAIContextAggregatorPair:
-        user = OpenAIUserContextAggregator(context)
-        assistant = OpenAIAssistantContextAggregator(
-            user, expect_stripped_words=assistant_expect_stripped_words
-        )
+        """Create an instance of OpenAIContextAggregatorPair from an
+        OpenAILLMContext. Constructor keyword arguments for both the user and
+        assistant aggregators can be provided.
+
+        Args:
+            context (OpenAILLMContext): The LLM context.
+            user_kwargs (Mapping[str, Any], optional): Additional keyword
+                arguments for the user context aggregator constructor. Defaults
+                to an empty mapping.
+            assistant_kwargs (Mapping[str, Any], optional): Additional keyword
+                arguments for the assistant context aggregator
+                constructor. Defaults to an empty mapping.
+
+        Returns:
+            OpenAIContextAggregatorPair: A pair of context aggregators, one for
+            the user and one for the assistant, encapsulated in an
+            OpenAIContextAggregatorPair.
+
+        """
+        context.set_llm_adapter(self.get_llm_adapter())
+        user = OpenAIUserContextAggregator(context, **user_kwargs)
+        assistant = OpenAIAssistantContextAggregator(context, **assistant_kwargs)
         return OpenAIContextAggregatorPair(_user=user, _assistant=assistant)
 
 
@@ -358,6 +391,7 @@ class OpenAIImageGenService(ImageGenService):
         self,
         *,
         api_key: str,
+        base_url: Optional[str] = None,
         aiohttp_session: aiohttp.ClientSession,
         image_size: Literal["256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"],
         model: str = "dall-e-3",
@@ -365,7 +399,7 @@ class OpenAIImageGenService(ImageGenService):
         super().__init__()
         self.set_model_name(model)
         self._image_size = image_size
-        self._client = AsyncOpenAI(api_key=api_key)
+        self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
         self._aiohttp_session = aiohttp_session
 
     async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
@@ -390,46 +424,102 @@ class OpenAIImageGenService(ImageGenService):
             yield frame
 
 
-class OpenAITTSService(TTSService):
-    """OpenAI Text-to-Speech service that generates audio from text.
+class OpenAISTTService(BaseWhisperSTTService):
+    """OpenAI Speech-to-Text service that generates text from audio.
 
-    This service uses the OpenAI TTS API to generate PCM-encoded audio at 24kHz.
-    When using with DailyTransport, configure the sample rate in DailyParams
-    as shown below:
-
-    DailyParams(
-        audio_out_enabled=True,
-        audio_out_sample_rate=24_000,
-    )
+    Uses OpenAI's transcription API to convert audio to text. Requires an OpenAI API key
+    set via the api_key parameter or OPENAI_API_KEY environment variable.
 
     Args:
+        model: Model to use — either gpt-4o or Whisper. Defaults to "gpt-4o-transcribe".
         api_key: OpenAI API key. Defaults to None.
-        voice: Voice ID to use. Defaults to "alloy".
-        model: TTS model to use ("tts-1" or "tts-1-hd"). Defaults to "tts-1".
-        sample_rate: Output audio sample rate in Hz. Defaults to 24000.
-        **kwargs: Additional keyword arguments passed to TTSService.
-
-    The service returns PCM-encoded audio at the specified sample rate.
+        base_url: API base URL. Defaults to None.
+        language: Language of the audio input. Defaults to English.
+        prompt: Optional text to guide the model's style or continue a previous segment.
+        temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0.
+        **kwargs: Additional arguments passed to BaseWhisperSTTService.
     """
 
     def __init__(
         self,
         *,
-        api_key: str | None = None,
-        voice: str = "alloy",
-        model: Literal["tts-1", "tts-1-hd"] = "tts-1",
-        sample_rate: int = 24000,
+        model: str = "gpt-4o-transcribe",
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        language: Optional[Language] = Language.EN,
+        prompt: Optional[str] = None,
+        temperature: Optional[float] = None,
         **kwargs,
     ):
+        super().__init__(
+            model=model,
+            api_key=api_key,
+            base_url=base_url,
+            language=language,
+            prompt=prompt,
+            temperature=temperature,
+            **kwargs,
+        )
+
+    async def _transcribe(self, audio: bytes) -> Transcription:
+        assert self._language is not None  # Assigned in the BaseWhisperSTTService class
+
+        # Build kwargs dict with only set parameters
+        kwargs = {
+            "file": ("audio.wav", audio, "audio/wav"),
+            "model": self.model_name,
+            "language": self._language,
+        }
+
+        if self._prompt is not None:
+            kwargs["prompt"] = self._prompt
+
+        if self._temperature is not None:
+            kwargs["temperature"] = self._temperature
+
+        return await self._client.audio.transcriptions.create(**kwargs)
+
+
+class OpenAITTSService(TTSService):
+    """OpenAI Text-to-Speech service that generates audio from text.
+
+    This service uses the OpenAI TTS API to generate PCM-encoded audio at 24kHz.
+
+    Args:
+        api_key: OpenAI API key. Defaults to None.
+        voice: Voice ID to use. Defaults to "alloy".
+        model: TTS model to use. Defaults to "gpt-4o-mini-tts".
+        sample_rate: Output audio sample rate in Hz. Defaults to None.
+        **kwargs: Additional keyword arguments passed to TTSService.
+
+    The service returns PCM-encoded audio at the specified sample rate.
+
+    """
+
+    OPENAI_SAMPLE_RATE = 24000  # OpenAI TTS always outputs at 24kHz
+
+    def __init__(
+        self,
+        *,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        voice: str = "alloy",
+        model: str = "gpt-4o-mini-tts",
+        sample_rate: Optional[int] = None,
+        instructions: Optional[str] = None,
+        **kwargs,
+    ):
+        if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE:
+            logger.warning(
+                f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
+                f"Current rate of {self.sample_rate}Hz may cause issues."
+            )
         super().__init__(sample_rate=sample_rate, **kwargs)
 
-        self._settings = {
-            "sample_rate": sample_rate,
-        }
         self.set_model_name(model)
         self.set_voice(voice)
-
-        self._client = AsyncOpenAI(api_key=api_key)
+        self._instructions = instructions
+        self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
 
     def can_generate_metrics(self) -> bool:
         return True
@@ -438,16 +528,30 @@ class OpenAITTSService(TTSService):
         logger.info(f"Switching TTS model to: [{model}]")
         self.set_model_name(model)
 
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+        if self.sample_rate != self.OPENAI_SAMPLE_RATE:
+            logger.warning(
+                f"OpenAI TTS requires {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
+                f"Current rate of {self.sample_rate}Hz may cause issues."
+            )
+
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
         try:
             await self.start_ttfb_metrics()
 
+            # Setup extra body parameters
+            extra_body = {}
+            if self._instructions:
+                extra_body["instructions"] = self._instructions
+
             async with self._client.audio.speech.with_streaming_response.create(
                 input=text or " ",  # Text must contain at least one character
                 model=self.model_name,
                 voice=VALID_VOICES[self._voice_id],
                 response_format="pcm",
+                extra_body=extra_body,
             ) as r:
                 if r.status_code != 200:
                     error = await r.text()
@@ -461,169 +565,80 @@ class OpenAITTSService(TTSService):
 
                 await self.start_tts_usage_metrics(text)
 
+                CHUNK_SIZE = 1024
+
                 yield TTSStartedFrame()
-                async for chunk in r.iter_bytes(8192):
+                async for chunk in r.iter_bytes(CHUNK_SIZE):
                     if len(chunk) > 0:
                         await self.stop_ttfb_metrics()
-                        frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
+                        frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
                         yield frame
                 yield TTSStoppedFrame()
         except BadRequestError as e:
             logger.exception(f"{self} error generating TTS: {e}")
 
 
-# internal use only -- todo: refactor
-@dataclass
-class OpenAIImageMessageFrame(Frame):
-    user_image_raw_frame: UserImageRawFrame
-    text: Optional[str] = None
-
-
 class OpenAIUserContextAggregator(LLMUserContextAggregator):
-    def __init__(self, context: OpenAILLMContext):
-        super().__init__(context=context)
-
-    async def process_frame(self, frame, direction):
-        await super().process_frame(frame, direction)
-        # Our parent method has already called push_frame(). So we can't interrupt the
-        # flow here and we don't need to call push_frame() ourselves.
-        try:
-            if isinstance(frame, UserImageRequestFrame):
-                # The LLM sends a UserImageRequestFrame upstream. Cache any context provided with
-                # that frame so we can use it when we assemble the image message in the assistant
-                # context aggregator.
-                if frame.context:
-                    if isinstance(frame.context, str):
-                        self._context._user_image_request_context[frame.user_id] = frame.context
-                    else:
-                        logger.error(
-                            f"Unexpected UserImageRequestFrame context type: {type(frame.context)}"
-                        )
-                        del self._context._user_image_request_context[frame.user_id]
-                else:
-                    if frame.user_id in self._context._user_image_request_context:
-                        del self._context._user_image_request_context[frame.user_id]
-            elif isinstance(frame, UserImageRawFrame):
-                # Push a new OpenAIImageMessageFrame with the text context we cached
-                # downstream to be handled by our assistant context aggregator. This is
-                # necessary so that we add the message to the context in the right order.
-                text = self._context._user_image_request_context.get(frame.user_id) or ""
-                if text:
-                    del self._context._user_image_request_context[frame.user_id]
-                frame = OpenAIImageMessageFrame(user_image_raw_frame=frame, text=text)
-                await self.push_frame(frame)
-        except Exception as e:
-            logger.error(f"Error processing frame: {e}")
+    pass
 
 
 class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
-    def __init__(self, user_context_aggregator: OpenAIUserContextAggregator, **kwargs):
-        super().__init__(context=user_context_aggregator._context, **kwargs)
-        self._user_context_aggregator = user_context_aggregator
-        self._function_calls_in_progress = {}
-        self._function_call_result = None
-        self._pending_image_frame_message = None
+    async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
+        self._context.add_message(
+            {
+                "role": "assistant",
+                "tool_calls": [
+                    {
+                        "id": frame.tool_call_id,
+                        "function": {
+                            "name": frame.function_name,
+                            "arguments": json.dumps(frame.arguments),
+                        },
+                        "type": "function",
+                    }
+                ],
+            }
+        )
+        self._context.add_message(
+            {
+                "role": "tool",
+                "content": "IN_PROGRESS",
+                "tool_call_id": frame.tool_call_id,
+            }
+        )
 
-    async def process_frame(self, frame, direction):
-        await super().process_frame(frame, direction)
-        # See note above about not calling push_frame() here.
-        if isinstance(frame, StartInterruptionFrame):
-            self._function_calls_in_progress.clear()
-            self._function_call_finished = None
-        elif isinstance(frame, FunctionCallInProgressFrame):
-            logger.debug(f"FunctionCallInProgressFrame: {frame}")
-            self._function_calls_in_progress[frame.tool_call_id] = frame
-        elif isinstance(frame, FunctionCallResultFrame):
-            logger.debug(f"FunctionCallResultFrame: {frame}")
-            if frame.tool_call_id in self._function_calls_in_progress:
-                del self._function_calls_in_progress[frame.tool_call_id]
-                self._function_call_result = frame
-                # TODO-CB: Kwin wants us to refactor this out of here but I REFUSE
-                await self._push_aggregation()
-            else:
-                logger.warning(
-                    "FunctionCallResultFrame tool_call_id does not match any function call in progress"
-                )
-                self._function_call_result = None
-        elif isinstance(frame, OpenAIImageMessageFrame):
-            self._pending_image_frame_message = frame
-            await self._push_aggregation()
+    async def handle_function_call_result(self, frame: FunctionCallResultFrame):
+        if frame.result:
+            result = json.dumps(frame.result)
+            await self._update_function_call_result(frame.function_name, frame.tool_call_id, result)
+        else:
+            await self._update_function_call_result(
+                frame.function_name, frame.tool_call_id, "COMPLETED"
+            )
 
-    async def _push_aggregation(self):
-        if not (
-            self._aggregation or self._function_call_result or self._pending_image_frame_message
-        ):
-            return
+    async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame):
+        await self._update_function_call_result(
+            frame.function_name, frame.tool_call_id, "CANCELLED"
+        )
 
-        run_llm = False
-        properties: Optional[FunctionCallResultProperties] = None
+    async def _update_function_call_result(
+        self, function_name: str, tool_call_id: str, result: Any
+    ):
+        for message in self._context.messages:
+            if (
+                message["role"] == "tool"
+                and message["tool_call_id"]
+                and message["tool_call_id"] == tool_call_id
+            ):
+                message["content"] = result
 
-        aggregation = self._aggregation
-        self._reset()
-
-        try:
-            if self._function_call_result:
-                frame = self._function_call_result
-                properties = frame.properties
-                self._function_call_result = None
-                if frame.result:
-                    self._context.add_message(
-                        {
-                            "role": "assistant",
-                            "tool_calls": [
-                                {
-                                    "id": frame.tool_call_id,
-                                    "function": {
-                                        "name": frame.function_name,
-                                        "arguments": json.dumps(frame.arguments),
-                                    },
-                                    "type": "function",
-                                }
-                            ],
-                        }
-                    )
-                    self._context.add_message(
-                        {
-                            "role": "tool",
-                            "content": json.dumps(frame.result),
-                            "tool_call_id": frame.tool_call_id,
-                        }
-                    )
-                    if properties and properties.run_llm is not None:
-                        # If the tool call result has a run_llm property, use it
-                        run_llm = properties.run_llm
-                    else:
-                        # Default behavior is to run the LLM if there are no function calls in progress
-                        run_llm = not bool(self._function_calls_in_progress)
-
-            else:
-                self._context.add_message({"role": "assistant", "content": aggregation})
-
-            if self._pending_image_frame_message:
-                frame = self._pending_image_frame_message
-                self._pending_image_frame_message = None
-                self._context.add_image_frame_message(
-                    format=frame.user_image_raw_frame.format,
-                    size=frame.user_image_raw_frame.size,
-                    image=frame.user_image_raw_frame.image,
-                    text=frame.text,
-                )
-                run_llm = True
-
-            if run_llm:
-                await self._user_context_aggregator.push_context_frame()
-
-            # Emit the on_context_updated callback once the function call result is added to the context
-            if properties and properties.on_context_updated is not None:
-                await properties.on_context_updated()
-
-            # Push context frame
-            frame = OpenAILLMContextFrame(self._context)
-            await self.push_frame(frame)
-
-            # Push timestamp frame with current time
-            timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601())
-            await self.push_frame(timestamp_frame)
-
-        except Exception as e:
-            logger.error(f"Error processing frame: {e}")
+    async def handle_user_image_frame(self, frame: UserImageRawFrame):
+        await self._update_function_call_result(
+            frame.request.function_name, frame.request.tool_call_id, "COMPLETED"
+        )
+        self._context.add_image_frame_message(
+            format=frame.format,
+            size=frame.size,
+            image=frame.image,
+            text=frame.request.context,
+        )

src/pipecat/services/openai_realtime_beta/__init__.py

@@ -1,2 +1,9 @@
-from .events import InputAudioTranscription, SessionProperties, TurnDetection
+from .azure import AzureRealtimeBetaLLMService
+from .events import (
+    InputAudioNoiseReduction,
+    InputAudioTranscription,
+    SemanticTurnDetection,
+    SessionProperties,
+    TurnDetection,
+)
 from .openai import OpenAIRealtimeBetaLLMService

src/pipecat/services/openai_realtime_beta/azure.py

@@ -0,0 +1,64 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+from loguru import logger
+
+from .openai import OpenAIRealtimeBetaLLMService
+
+try:
+    import websockets
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable."
+    )
+    raise Exception(f"Missing module: {e}")
+
+
+class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService):
+    """Subclass of OpenAI Realtime API Service with adjustments for Azure's wss connection."""
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        base_url: str,
+        **kwargs,
+    ):
+        """Constructor takes the same arguments as the parent class, OpenAIRealtimeBetaLLMService.
+
+        Note that the following are required arguments:
+            api_key: The API key for the Azure OpenAI service.
+            base_url: The base URL for the Azure OpenAI service.
+
+        base_url should be set to the full Azure endpoint URL including the api-version and the deployment name. For example,
+
+        wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment
+        """
+        super().__init__(base_url=base_url, api_key=api_key, **kwargs)
+        self.api_key = api_key
+        self.base_url = base_url
+
+    async def _connect(self):
+        try:
+            if self._websocket:
+                # Here we assume that if we have a websocket, we are connected. We
+                # handle disconnections in the send/recv code paths.
+                return
+
+            logger.info(f"Connecting to {self.base_url}, api key: {self.api_key}")
+            self._websocket = await websockets.connect(
+                uri=self.base_url,
+                extra_headers={
+                    "api-key": self.api_key,
+                },
+            )
+            self._receive_task = self.create_task(self._receive_task_handler())
+        except Exception as e:
+            logger.error(f"{self} initialization error: {e}")
+            self._websocket = None

src/pipecat/services/openai_realtime_beta/context.py

@@ -12,6 +12,7 @@ from loguru import logger
 
 from pipecat.frames.frames import (
     Frame,
+    FunctionCallResultFrame,
     FunctionCallResultProperties,
     LLMMessagesUpdateFrame,
     LLMSetToolsFrame,
@@ -166,7 +167,7 @@ class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator):
         if isinstance(frame, LLMSetToolsFrame):
             await self.push_frame(frame, direction)
 
-    async def _push_aggregation(self):
+    async def push_aggregation(self):
         # for the moment, ignore all user input coming into the pipeline.
         # todo: think about whether/how to fix this to allow for text input from
         #       upstream (transport/transcription, or other sources)
@@ -174,68 +175,12 @@ class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator):
 
 
 class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator):
-    async def _push_aggregation(self):
-        # the only thing we implement here is function calling. in all other cases, messages
-        # are added to the context when we receive openai realtime api events
-        if not self._function_call_result:
-            return
+    async def handle_function_call_result(self, frame: FunctionCallResultFrame):
+        await super().handle_function_call_result(frame)
 
-        properties: Optional[FunctionCallResultProperties] = None
-
-        self._reset()
-        try:
-            run_llm = True
-            frame = self._function_call_result
-            properties = frame.properties
-            self._function_call_result = None
-            if frame.result:
-                # The "tool_call" message from the LLM that triggered the function call
-                self._context.add_message(
-                    {
-                        "role": "assistant",
-                        "tool_calls": [
-                            {
-                                "id": frame.tool_call_id,
-                                "function": {
-                                    "name": frame.function_name,
-                                    "arguments": json.dumps(frame.arguments),
-                                },
-                                "type": "function",
-                            }
-                        ],
-                    }
-                )
-                # The result of the function call. Need to add this both to our context here and to
-                # the openai realtime api context.
-                result_message = {
-                    "role": "tool",
-                    "content": json.dumps(frame.result),
-                    "tool_call_id": frame.tool_call_id,
-                }
-
-                self._context.add_message(result_message)
-                # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself,
-                # so we didn't have a chance to add the result to the openai realtime api context. Let's push a
-                # special frame to do that.
-                await self._user_context_aggregator.push_frame(
-                    RealtimeFunctionCallResultFrame(result_frame=frame)
-                )
-                if properties and properties.run_llm is not None:
-                    # If the tool call result has a run_llm property, use it
-                    run_llm = properties.run_llm
-                else:
-                    # Default behavior is to run the LLM if there are no function calls in progress
-                    run_llm = not bool(self._function_calls_in_progress)
-
-            if run_llm:
-                await self._user_context_aggregator.push_context_frame()
-
-            # Emit the on_context_updated callback once the function call result is added to the context
-            if properties and properties.on_context_updated is not None:
-                await properties.on_context_updated()
-
-            frame = OpenAILLMContextFrame(self._context)
-            await self.push_frame(frame)
-
-        except Exception as e:
-            logger.error(f"Error processing frame: {e}")
+        # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself,
+        # so we didn't have a chance to add the result to the openai realtime api context. Let's push a
+        # special frame to do that.
+        await self.push_frame(
+            RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM
+        )

src/pipecat/services/openai_realtime_beta/events.py

@@ -17,7 +17,29 @@ from pydantic import BaseModel, Field
 
 
 class InputAudioTranscription(BaseModel):
-    model: Optional[str] = "whisper-1"
+    """Configuration for audio transcription settings.
+
+    Attributes:
+        model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1").
+        language: Optional language code for transcription.
+        prompt: Optional transcription hint text.
+    """
+
+    model: str = "gpt-4o-transcribe"
+    language: Optional[str]
+    prompt: Optional[str]
+
+    def __init__(
+        self,
+        model: Optional[str] = "gpt-4o-transcribe",
+        language: Optional[str] = None,
+        prompt: Optional[str] = None,
+    ):
+        super().__init__(model=model, language=language, prompt=prompt)
+        if self.model != "gpt-4o-transcribe" and (self.language or self.prompt):
+            raise ValueError(
+                "Fields 'language' and 'prompt' are only supported when model is 'gpt-4o-transcribe'"
+            )
 
 
 class TurnDetection(BaseModel):
@@ -27,6 +49,17 @@ class TurnDetection(BaseModel):
     silence_duration_ms: Optional[int] = 800
 
 
+class SemanticTurnDetection(BaseModel):
+    type: Optional[Literal["semantic_vad"]] = "semantic_vad"
+    eagerness: Optional[Literal["low", "medium", "high", "auto"]] = None
+    create_response: Optional[bool] = None
+    interrupt_response: Optional[bool] = None
+
+
+class InputAudioNoiseReduction(BaseModel):
+    type: Optional[Literal["near_field", "far_field"]]
+
+
 class SessionProperties(BaseModel):
     modalities: Optional[List[Literal["text", "audio"]]] = None
     instructions: Optional[str] = None
@@ -34,8 +67,11 @@ class SessionProperties(BaseModel):
     input_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None
     output_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None
     input_audio_transcription: Optional[InputAudioTranscription] = None
+    input_audio_noise_reduction: Optional[InputAudioNoiseReduction] = None
     # set turn_detection to False to disable turn detection
-    turn_detection: Optional[Union[TurnDetection, bool]] = Field(default=None)
+    turn_detection: Optional[Union[TurnDetection, SemanticTurnDetection, bool]] = Field(
+        default=None
+    )
     tools: Optional[List[Dict]] = None
     tool_choice: Optional[Literal["auto", "none", "required"]] = None
     temperature: Optional[float] = None
@@ -93,6 +129,7 @@ class RealtimeError(BaseModel):
     code: Optional[str] = ""
     message: str
     param: Optional[str] = None
+    event_id: Optional[str] = None
 
 
 #
@@ -150,6 +187,11 @@ class ConversationItemDeleteEvent(ClientEvent):
     item_id: str
 
 
+class ConversationItemRetrieveEvent(ClientEvent):
+    type: Literal["conversation.item.retrieve"] = "conversation.item.retrieve"
+    item_id: str
+
+
 class ResponseCreateEvent(ClientEvent):
     type: Literal["response.create"] = "response.create"
     response: Optional[ResponseProperties] = None
@@ -193,6 +235,13 @@ class ConversationItemCreated(ServerEvent):
     item: ConversationItem
 
 
+class ConversationItemInputAudioTranscriptionDelta(ServerEvent):
+    type: Literal["conversation.item.input_audio_transcription.delta"]
+    item_id: str
+    content_index: int
+    delta: str
+
+
 class ConversationItemInputAudioTranscriptionCompleted(ServerEvent):
     type: Literal["conversation.item.input_audio_transcription.completed"]
     item_id: str
@@ -219,6 +268,11 @@ class ConversationItemDeleted(ServerEvent):
     item_id: str
 
 
+class ConversationItemRetrieved(ServerEvent):
+    type: Literal["conversation.item.retrieved"]
+    item: ConversationItem
+
+
 class ResponseCreated(ServerEvent):
     type: Literal["response.created"]
     response: "Response"
@@ -400,10 +454,12 @@ _server_event_types = {
     "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted,
     "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped,
     "conversation.item.created": ConversationItemCreated,
+    "conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta,
     "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted,
     "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed,
     "conversation.item.truncated": ConversationItemTruncated,
     "conversation.item.deleted": ConversationItemDeleted,
+    "conversation.item.retrieved": ConversationItemRetrieved,
     "response.created": ResponseCreated,
     "response.done": ResponseDone,
     "response.output_item.added": ResponseOutputItemAdded,

src/pipecat/services/openai_realtime_beta/openai.py

@@ -4,15 +4,25 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-import asyncio
 import base64
 import json
 import time
 from dataclasses import dataclass
+from typing import Any, Mapping
 
-import websockets
 from loguru import logger
 
+from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter
+
+try:
+    import websockets
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable."
+    )
+    raise Exception(f"Missing module: {e}")
+
 from pipecat.frames.frames import (
     BotStoppedSpeakingFrame,
     CancelFrame,
@@ -20,6 +30,7 @@ from pipecat.frames.frames import (
     ErrorFrame,
     Frame,
     InputAudioRawFrame,
+    InterimTranscriptionFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesAppendFrame,
@@ -33,6 +44,7 @@ from pipecat.frames.frames import (
     TTSAudioRawFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
+    TTSTextFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
 )
@@ -68,6 +80,9 @@ class OpenAIUnhandledFunctionException(Exception):
 
 
 class OpenAIRealtimeBetaLLMService(LLMService):
+    # Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one.
+    adapter_class = OpenAIRealtimeLLMAdapter
+
     def __init__(
         self,
         *,
@@ -101,12 +116,35 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         self._messages_added_manually = {}
         self._user_and_response_message_tuple = None
 
+        self._register_event_handler("on_conversation_item_created")
+        self._register_event_handler("on_conversation_item_updated")
+        self._retrieve_conversation_item_futures = {}
+
     def can_generate_metrics(self) -> bool:
         return True
 
     def set_audio_input_paused(self, paused: bool):
         self._audio_input_paused = paused
 
+    async def retrieve_conversation_item(self, item_id: str):
+        future = self.get_event_loop().create_future()
+        retrieval_in_flight = False
+        if not self._retrieve_conversation_item_futures.get(item_id):
+            self._retrieve_conversation_item_futures[item_id] = []
+        else:
+            retrieval_in_flight = True
+        self._retrieve_conversation_item_futures[item_id].append(future)
+        if not retrieval_in_flight:
+            await self.send_client_event(
+                # Set event_id to "rci_{item_id}" so that we can identify an
+                # error later if the retrieval fails. We don't need a UUID
+                # suffix to the event_id because we're ensuring only one
+                # in-flight retrieval per item_id. (Note: "rci" = "retrieve
+                # conversation item")
+                events.ConversationItemRetrieveEvent(item_id=item_id, event_id=f"rci_{item_id}")
+            )
+        return await future
+
     #
     # standard AIService frame handling
     #
@@ -340,8 +378,12 @@ class OpenAIRealtimeBetaLLMService(LLMService):
                 await self._handle_evt_audio_done(evt)
             elif evt.type == "conversation.item.created":
                 await self._handle_evt_conversation_item_created(evt)
+            elif evt.type == "conversation.item.input_audio_transcription.delta":
+                await self._handle_evt_input_audio_transcription_delta(evt)
             elif evt.type == "conversation.item.input_audio_transcription.completed":
                 await self.handle_evt_input_audio_transcription_completed(evt)
+            elif evt.type == "conversation.item.retrieved":
+                await self._handle_conversation_item_retrieved(evt)
             elif evt.type == "response.done":
                 await self._handle_evt_response_done(evt)
             elif evt.type == "input_audio_buffer.speech_started":
@@ -351,9 +393,10 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             elif evt.type == "response.audio_transcript.delta":
                 await self._handle_evt_audio_transcript_delta(evt)
             elif evt.type == "error":
-                await self._handle_evt_error(evt)
-                # errors are fatal, so exit the receive loop
-                return
+                if not await self._maybe_handle_evt_retrieve_conversation_item_error(evt):
+                    await self._handle_evt_error(evt)
+                    # errors are fatal, so exit the receive loop
+                    return
 
     async def _handle_evt_session_created(self, evt):
         # session.created is received right after connecting. Send a message
@@ -395,6 +438,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             # receive a BotStoppedSpeakingFrame from the output transport.
 
     async def _handle_evt_conversation_item_created(self, evt):
+        await self._call_event_handler("on_conversation_item_created", evt.item.id, evt.item)
+
         # This will get sent from the server every time a new "message" is added
         # to the server's conversation state, whether we create it via the API
         # or the server creates it from LLM output.
@@ -411,7 +456,16 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             self._current_assistant_response = evt.item
             await self.push_frame(LLMFullResponseStartFrame())
 
+    async def _handle_evt_input_audio_transcription_delta(self, evt):
+        if self._send_transcription_frames:
+            await self.push_frame(
+                # no way to get a language code?
+                InterimTranscriptionFrame(evt.delta, "", time_now_iso8601())
+            )
+
     async def handle_evt_input_audio_transcription_completed(self, evt):
+        await self._call_event_handler("on_conversation_item_updated", evt.item_id, None)
+
         if self._send_transcription_frames:
             await self.push_frame(
                 # no way to get a language code?
@@ -429,6 +483,12 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             # User message without preceding conversation.item.created. Bug?
             logger.warning(f"Transcript for unknown user message: {evt}")
 
+    async def _handle_conversation_item_retrieved(self, evt: events.ConversationItemRetrieved):
+        futures = self._retrieve_conversation_item_futures.pop(evt.item.id, None)
+        if futures:
+            for future in futures:
+                future.set_result(evt.item)
+
     async def _handle_evt_response_done(self, evt):
         # todo: figure out whether there's anything we need to do for "cancelled" events
         # usage metrics
@@ -441,7 +501,15 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         await self.stop_processing_metrics()
         await self.push_frame(LLMFullResponseEndFrame())
         self._current_assistant_response = None
+        # error handling
+        if evt.response.status == "failed":
+            await self.push_error(
+                ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True)
+            )
+            return
         # response content
+        for item in evt.response.output:
+            await self._call_event_handler("on_conversation_item_updated", item.id, item)
         pair = self._user_and_response_message_tuple
         if pair:
             user, assistant = pair
@@ -458,6 +526,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
     async def _handle_evt_audio_transcript_delta(self, evt):
         if evt.delta:
             await self.push_frame(LLMTextFrame(evt.delta))
+            await self.push_frame(TTSTextFrame(evt.delta))
 
     async def _handle_evt_speech_started(self, evt):
         await self._truncate_current_audio_response()
@@ -472,6 +541,22 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         await self.push_frame(StopInterruptionFrame())
         await self.push_frame(UserStoppedSpeakingFrame())
 
+    async def _maybe_handle_evt_retrieve_conversation_item_error(self, evt: events.ErrorEvent):
+        """If the given error event is an error retrieving a conversation item:
+        - set an exception on the future that retrieve_conversation_item() is waiting on
+        - return true
+        Otherwise:
+        - return false
+        """
+        if evt.error.code == "item_retrieve_invalid_item_id":
+            item_id = evt.error.event_id.split("_", 1)[1]  # event_id is of the form "rci_{item_id}"
+            futures = self._retrieve_conversation_item_futures.pop(item_id, None)
+            if futures:
+                for future in futures:
+                    future.set_exception(Exception(evt.error.message))
+            return True
+        return False
+
     async def _handle_evt_error(self, evt):
         # Errors are fatal to this connection. Send an ErrorFrame.
         await self.push_error(ErrorFrame(error=f"Error: {evt}", fatal=True))
@@ -494,7 +579,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             arguments = json.loads(item.arguments)
             if self.has_function(function_name):
                 run_llm = index == total_items - 1
-                if function_name in self._callbacks.keys():
+                if function_name in self._functions.keys():
                     await self.call_function(
                         context=self._context,
                         tool_call_id=tool_id,
@@ -502,7 +587,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
                         arguments=arguments,
                         run_llm=run_llm,
                     )
-                elif None in self._callbacks.keys():
+                elif None in self._functions.keys():
                     await self.call_function(
                         context=self._context,
                         tool_call_id=tool_id,
@@ -563,11 +648,37 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload))
 
     def create_context_aggregator(
-        self, context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = False
+        self,
+        context: OpenAILLMContext,
+        *,
+        user_kwargs: Mapping[str, Any] = {},
+        assistant_kwargs: Mapping[str, Any] = {},
     ) -> OpenAIContextAggregatorPair:
+        """Create an instance of OpenAIContextAggregatorPair from an
+        OpenAILLMContext. Constructor keyword arguments for both the user and
+        assistant aggregators can be provided.
+
+        Args:
+            context (OpenAILLMContext): The LLM context.
+            user_kwargs (Mapping[str, Any], optional): Additional keyword
+                arguments for the user context aggregator constructor. Defaults
+                to an empty mapping.
+            assistant_kwargs (Mapping[str, Any], optional): Additional keyword
+                arguments for the assistant context aggregator
+                constructor. Defaults to an empty mapping.
+
+        Returns:
+            OpenAIContextAggregatorPair: A pair of context aggregators, one for
+            the user and one for the assistant, encapsulated in an
+            OpenAIContextAggregatorPair.
+
+        """
+        context.set_llm_adapter(self.get_llm_adapter())
+
         OpenAIRealtimeLLMContext.upgrade_to_realtime(context)
-        user = OpenAIRealtimeUserContextAggregator(context)
-        assistant = OpenAIRealtimeAssistantContextAggregator(
-            user, expect_stripped_words=assistant_expect_stripped_words
-        )
+        user = OpenAIRealtimeUserContextAggregator(context, **user_kwargs)
+
+        default_assistant_kwargs = {"expect_stripped_words": False}
+        default_assistant_kwargs.update(assistant_kwargs)
+        assistant = OpenAIRealtimeAssistantContextAggregator(context, **default_assistant_kwargs)
         return OpenAIContextAggregatorPair(_user=user, _assistant=assistant)

src/pipecat/services/openpipe.py

@@ -4,15 +4,15 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 from loguru import logger
+from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
 
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai import OpenAILLMService
 
 try:
-    from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
     from openpipe import AsyncOpenAI as OpenPipeAI
     from openpipe import AsyncStream
 except ModuleNotFoundError as e:
@@ -28,11 +28,11 @@ class OpenPipeLLMService(OpenAILLMService):
         self,
         *,
         model: str = "gpt-4o",
-        api_key: str | None = None,
-        base_url: str | None = None,
-        openpipe_api_key: str | None = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        openpipe_api_key: Optional[str] = None,
         openpipe_base_url: str = "https://app.openpipe.ai/api/v1",
-        tags: Dict[str, str] | None = None,
+        tags: Optional[Dict[str, str]] = None,
         **kwargs,
     ):
         super().__init__(

src/pipecat/services/openrouter.py

@@ -4,23 +4,12 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-from typing import Dict, List
+from typing import Optional
 
 from loguru import logger
 
-from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai import OpenAILLMService
 
-try:
-    from openai import AsyncStream, OpenAI
-    from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error(
-        "In order to use OpenRouter, you need to `pip install pipecat-ai[openrouter]`. Also, set `OPENROUTER_API_KEY` environment variable."
-    )
-    raise Exception(f"Missing module: {e}")
-
 
 class OpenRouterLLMService(OpenAILLMService):
     """A service for interacting with OpenRouter's API using the OpenAI-compatible interface.
@@ -38,7 +27,7 @@ class OpenRouterLLMService(OpenAILLMService):
     def __init__(
         self,
         *,
-        api_key: str | None = None,
+        api_key: Optional[str] = None,
         model: str = "openai/gpt-4o-2024-11-20",
         base_url: str = "https://openrouter.ai/api/v1",
         **kwargs,

src/pipecat/services/perplexity.py

@@ -0,0 +1,130 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from typing import List
+
+from loguru import logger
+from openai import NOT_GIVEN, AsyncStream
+from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
+
+from pipecat.metrics.metrics import LLMTokenUsage
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.openai import OpenAILLMService
+
+
+class PerplexityLLMService(OpenAILLMService):
+    """A service for interacting with Perplexity's API.
+
+    This service extends OpenAILLMService to work with Perplexity's API while maintaining
+    compatibility with the OpenAI-style interface. It specifically handles the difference
+    in token usage reporting between Perplexity (incremental) and OpenAI (final summary).
+
+    Args:
+        api_key (str): The API key for accessing Perplexity's API
+        base_url (str, optional): The base URL for Perplexity's API. Defaults to "https://api.perplexity.ai"
+        model (str, optional): The model identifier to use. Defaults to "sonar"
+        **kwargs: Additional keyword arguments passed to OpenAILLMService
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        base_url: str = "https://api.perplexity.ai",
+        model: str = "sonar",
+        **kwargs,
+    ):
+        super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
+        # Counters for accumulating token usage metrics
+        self._prompt_tokens = 0
+        self._completion_tokens = 0
+        self._total_tokens = 0
+        self._has_reported_prompt_tokens = False
+        self._is_processing = False
+
+    async def get_chat_completions(
+        self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam]
+    ) -> AsyncStream[ChatCompletionChunk]:
+        """Get chat completions from Perplexity API using OpenAI-compatible parameters.
+
+        Args:
+            context: The context containing conversation history and settings
+            messages: The messages to send to the API
+
+        Returns:
+            A stream of chat completion chunks
+        """
+        params = {
+            "model": self.model_name,
+            "stream": True,
+            "messages": messages,
+        }
+
+        # Add OpenAI-compatible parameters if they're set
+        if self._settings["frequency_penalty"] is not NOT_GIVEN:
+            params["frequency_penalty"] = self._settings["frequency_penalty"]
+        if self._settings["presence_penalty"] is not NOT_GIVEN:
+            params["presence_penalty"] = self._settings["presence_penalty"]
+        if self._settings["temperature"] is not NOT_GIVEN:
+            params["temperature"] = self._settings["temperature"]
+        if self._settings["top_p"] is not NOT_GIVEN:
+            params["top_p"] = self._settings["top_p"]
+        if self._settings["max_tokens"] is not NOT_GIVEN:
+            params["max_tokens"] = self._settings["max_tokens"]
+
+        chunks = await self._client.chat.completions.create(**params)
+        return chunks
+
+    async def _process_context(self, context: OpenAILLMContext):
+        """Process a context through the LLM and accumulate token usage metrics.
+
+        This method overrides the parent class implementation to handle
+        Perplexity's incremental token reporting style, accumulating the counts
+        and reporting them once at the end of processing.
+
+        Args:
+            context (OpenAILLMContext): The context to process, containing messages
+                and other information needed for the LLM interaction.
+        """
+        # Reset all counters and flags at the start of processing
+        self._prompt_tokens = 0
+        self._completion_tokens = 0
+        self._total_tokens = 0
+        self._has_reported_prompt_tokens = False
+        self._is_processing = True
+
+        try:
+            await super()._process_context(context)
+        finally:
+            self._is_processing = False
+            # Report final accumulated token usage at the end of processing
+            if self._prompt_tokens > 0 or self._completion_tokens > 0:
+                self._total_tokens = self._prompt_tokens + self._completion_tokens
+                tokens = LLMTokenUsage(
+                    prompt_tokens=self._prompt_tokens,
+                    completion_tokens=self._completion_tokens,
+                    total_tokens=self._total_tokens,
+                )
+                await super().start_llm_usage_metrics(tokens)
+
+    async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
+        """Accumulate token usage metrics during processing.
+
+        Perplexity reports token usage incrementally during streaming,
+        unlike OpenAI which provides a final summary. We accumulate the
+        counts and report the total at the end of processing.
+        """
+        if not self._is_processing:
+            return
+
+        # Record prompt tokens the first time we see them
+        if not self._has_reported_prompt_tokens and tokens.prompt_tokens > 0:
+            self._prompt_tokens = tokens.prompt_tokens
+            self._has_reported_prompt_tokens = True
+
+        # Update completion tokens count if it has increased
+        if tokens.completion_tokens > self._completion_tokens:
+            self._completion_tokens = tokens.completion_tokens

src/pipecat/services/playht.py

@@ -16,22 +16,18 @@ from loguru import logger
 from pydantic import BaseModel
 
 from pipecat.frames.frames import (
-    BotStoppedSpeakingFrame,
     CancelFrame,
     EndFrame,
     ErrorFrame,
     Frame,
-    LLMFullResponseEndFrame,
     StartFrame,
     StartInterruptionFrame,
     TTSAudioRawFrame,
-    TTSSpeakFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.ai_services import TTSService
-from pipecat.services.websocket_service import WebsocketService
+from pipecat.services.ai_services import InterruptibleTTSService, TTSService
 from pipecat.transcriptions.language import Language
 
 try:
@@ -46,7 +42,7 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
-def language_to_playht_language(language: Language) -> str | None:
+def language_to_playht_language(language: Language) -> Optional[str]:
     BASE_LANGUAGES = {
         Language.AF: "afrikans",
         Language.AM: "amharic",
@@ -100,7 +96,7 @@ def language_to_playht_language(language: Language) -> str | None:
     return result
 
 
-class PlayHTTTSService(TTSService, WebsocketService):
+class PlayHTTTSService(InterruptibleTTSService):
     class InputParams(BaseModel):
         language: Optional[Language] = Language.EN
         speed: Optional[float] = 1.0
@@ -113,17 +109,16 @@ class PlayHTTTSService(TTSService, WebsocketService):
         user_id: str,
         voice_url: str,
         voice_engine: str = "Play3.0-mini",
-        sample_rate: int = 24000,
+        sample_rate: Optional[int] = None,
         output_format: str = "wav",
         params: InputParams = InputParams(),
         **kwargs,
     ):
-        TTSService.__init__(
-            self,
+        super().__init__(
+            pause_frame_processing=True,
             sample_rate=sample_rate,
             **kwargs,
         )
-        WebsocketService.__init__(self)
 
         self._api_key = api_key
         self._user_id = user_id
@@ -132,7 +127,6 @@ class PlayHTTTSService(TTSService, WebsocketService):
         self._request_id = None
 
         self._settings = {
-            "sample_rate": sample_rate,
             "language": self.language_to_service_language(params.language)
             if params.language
             else "english",
@@ -147,7 +141,7 @@ class PlayHTTTSService(TTSService, WebsocketService):
     def can_generate_metrics(self) -> bool:
         return True
 
-    def language_to_service_language(self, language: Language) -> str | None:
+    def language_to_service_language(self, language: Language) -> Optional[str]:
         return language_to_playht_language(language)
 
     async def start(self, frame: StartFrame):
@@ -165,17 +159,21 @@ class PlayHTTTSService(TTSService, WebsocketService):
     async def _connect(self):
         await self._connect_websocket()
 
-        self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
+        if not self._receive_task:
+            self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
 
     async def _disconnect(self):
-        await self._disconnect_websocket()
-
         if self._receive_task:
             await self.cancel_task(self._receive_task)
             self._receive_task = None
 
+        await self._disconnect_websocket()
+
     async def _connect_websocket(self):
         try:
+            if self._websocket:
+                return
+
             logger.debug("Connecting to PlayHT")
 
             if not self._websocket_url:
@@ -185,12 +183,14 @@ class PlayHTTTSService(TTSService, WebsocketService):
                 raise ValueError("WebSocket URL is not a string")
 
             self._websocket = await websockets.connect(self._websocket_url)
-        except ValueError as ve:
-            logger.error(f"{self} initialization error: {ve}")
+        except ValueError as e:
+            logger.error(f"{self} initialization error: {e}")
             self._websocket = None
+            await self._call_event_handler("on_connection_error", f"{e}")
         except Exception as e:
             logger.error(f"{self} initialization error: {e}")
             self._websocket = None
+            await self._call_event_handler("on_connection_error", f"{e}")
 
     async def _disconnect_websocket(self):
         try:
@@ -250,7 +250,7 @@ class PlayHTTTSService(TTSService, WebsocketService):
                 if message.startswith(b"RIFF"):
                     continue
                 await self.stop_ttfb_metrics()
-                frame = TTSAudioRawFrame(message, self._settings["sample_rate"], 1)
+                frame = TTSAudioRawFrame(message, self.sample_rate, 1)
                 await self.push_frame(frame)
             else:
                 logger.debug(f"Received text message: {message}")
@@ -270,21 +270,8 @@ class PlayHTTTSService(TTSService, WebsocketService):
                 except json.JSONDecodeError:
                     logger.error(f"Invalid JSON message: {message}")
 
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        # If we received a TTSSpeakFrame and the LLM response included text (it
-        # might be that it's only a function calling response) we pause
-        # processing more frames until we receive a BotStoppedSpeakingFrame.
-        if isinstance(frame, TTSSpeakFrame):
-            await self.pause_processing_frames()
-        elif isinstance(frame, LLMFullResponseEndFrame) and self._request_id:
-            await self.pause_processing_frames()
-        elif isinstance(frame, BotStoppedSpeakingFrame):
-            await self.resume_processing_frames()
-
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
             # Reconnect if the websocket is closed
@@ -301,7 +288,7 @@ class PlayHTTTSService(TTSService, WebsocketService):
                 "voice": self._voice_id,
                 "voice_engine": self._settings["voice_engine"],
                 "output_format": self._settings["output_format"],
-                "sample_rate": self._settings["sample_rate"],
+                "sample_rate": self.sample_rate,
                 "language": self._settings["language"],
                 "speed": self._settings["speed"],
                 "seed": self._settings["seed"],
@@ -338,8 +325,9 @@ class PlayHTHttpTTSService(TTSService):
         api_key: str,
         user_id: str,
         voice_url: str,
-        voice_engine: str = "Play3.0-mini-http",  # Options: Play3.0-mini-http, Play3.0-mini-ws
-        sample_rate: int = 24000,
+        voice_engine: str = "Play3.0-mini",
+        protocol: str = "http",  # Options: http, ws
+        sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
     ):
@@ -352,19 +340,35 @@ class PlayHTHttpTTSService(TTSService):
             user_id=self._user_id,
             api_key=self._api_key,
         )
+
+        # Check if voice_engine contains protocol information (backward compatibility)
+        if "-http" in voice_engine:
+            # Extract the base engine name
+            voice_engine = voice_engine.replace("-http", "")
+            protocol = "http"
+        elif "-ws" in voice_engine:
+            # Extract the base engine name
+            voice_engine = voice_engine.replace("-ws", "")
+            protocol = "ws"
+
         self._settings = {
-            "sample_rate": sample_rate,
             "language": self.language_to_service_language(params.language)
             if params.language
             else "english",
             "format": Format.FORMAT_WAV,
             "voice_engine": voice_engine,
+            "protocol": protocol,
             "speed": params.speed,
             "seed": params.seed,
         }
         self.set_model_name(voice_engine)
         self.set_voice(voice_url)
 
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+        self._settings["sample_rate"] = self.sample_rate
+
+    def _create_options(self) -> TTSOptions:
         language_str = self._settings["language"]
         playht_language = None
         if language_str:
@@ -374,10 +378,10 @@ class PlayHTHttpTTSService(TTSService):
                     playht_language = lang
                     break
 
-        self._options = TTSOptions(
+        return TTSOptions(
             voice=self._voice_id,
             language=playht_language,
-            sample_rate=self._settings["sample_rate"],
+            sample_rate=self.sample_rate,
             format=self._settings["format"],
             speed=self._settings["speed"],
             seed=self._settings["seed"],
@@ -386,25 +390,30 @@ class PlayHTHttpTTSService(TTSService):
     def can_generate_metrics(self) -> bool:
         return True
 
-    def language_to_service_language(self, language: Language) -> str | None:
+    def language_to_service_language(self, language: Language) -> Optional[str]:
         return language_to_playht_language(language)
 
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
-            b = bytearray()
-            in_header = True
+            options = self._create_options()
 
             await self.start_ttfb_metrics()
 
             playht_gen = self._client.tts(
-                text, voice_engine=self._settings["voice_engine"], options=self._options
+                text,
+                voice_engine=self._settings["voice_engine"],
+                protocol=self._settings["protocol"],
+                options=options,
             )
 
             await self.start_tts_usage_metrics(text)
 
             yield TTSStartedFrame()
+
+            b = bytearray()
+            in_header = True
             async for chunk in playht_gen:
                 # skip the RIFF header.
                 if in_header:
@@ -419,11 +428,12 @@ class PlayHTHttpTTSService(TTSService):
                             fh.read(size)
                             (data, size) = struct.unpack("<4sI", fh.read(8))
                         in_header = False
-                else:
-                    if len(chunk):
-                        await self.stop_ttfb_metrics()
-                        frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
-                        yield frame
-            yield TTSStoppedFrame()
+                elif len(chunk) > 0:
+                    await self.stop_ttfb_metrics()
+                    frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
+                    yield frame
         except Exception as e:
             logger.error(f"{self} error generating TTS: {e}")
+        finally:
+            await self.stop_ttfb_metrics()
+            yield TTSStoppedFrame()

src/pipecat/services/rime.py

@@ -4,6 +4,9 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+import base64
+import json
+import uuid
 from typing import AsyncGenerator, Optional
 
 import aiohttp
@@ -11,13 +14,333 @@ from loguru import logger
 from pydantic import BaseModel
 
 from pipecat.frames.frames import (
+    CancelFrame,
+    EndFrame,
     ErrorFrame,
     Frame,
+    StartFrame,
+    StartInterruptionFrame,
     TTSAudioRawFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
 )
-from pipecat.services.ai_services import TTSService
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.ai_services import AudioContextWordTTSService, TTSService
+from pipecat.transcriptions.language import Language
+from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
+from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator
+
+try:
+    import websockets
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use Rime, you need to `pip install pipecat-ai[rime]`. Also, set `RIME_API_KEY` environment variable."
+    )
+    raise Exception(f"Missing module: {e}")
+
+
+def language_to_rime_language(language: Language) -> str:
+    """Convert pipecat Language to Rime language code.
+
+    Args:
+        language: The pipecat Language enum value.
+
+    Returns:
+        str: Three-letter language code used by Rime (e.g., 'eng' for English).
+    """
+    LANGUAGE_MAP = {
+        Language.EN: "eng",
+        Language.ES: "spa",
+    }
+    return LANGUAGE_MAP.get(language, "eng")
+
+
+class RimeTTSService(AudioContextWordTTSService):
+    """Text-to-Speech service using Rime's websocket API.
+
+    Uses Rime's websocket JSON API to convert text to speech with word-level timing
+    information. Supports interruptions and maintains context across multiple messages
+    within a turn.
+    """
+
+    class InputParams(BaseModel):
+        """Configuration parameters for Rime TTS service."""
+
+        language: Optional[Language] = Language.EN
+        speed_alpha: Optional[float] = 1.0
+        reduce_latency: Optional[bool] = False
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        voice_id: str,
+        url: str = "wss://users-ws.rime.ai/ws2",
+        model: str = "mistv2",
+        sample_rate: Optional[int] = None,
+        params: InputParams = InputParams(),
+        text_aggregator: Optional[BaseTextAggregator] = None,
+        **kwargs,
+    ):
+        """Initialize Rime TTS service.
+
+        Args:
+            api_key: Rime API key for authentication.
+            voice_id: ID of the voice to use.
+            url: Rime websocket API endpoint.
+            model: Model ID to use for synthesis.
+            sample_rate: Audio sample rate in Hz.
+            params: Additional configuration parameters.
+        """
+        # Initialize with parent class settings for proper frame handling
+        super().__init__(
+            aggregate_sentences=True,
+            push_text_frames=False,
+            push_stop_frames=True,
+            pause_frame_processing=True,
+            sample_rate=sample_rate,
+            text_aggregator=text_aggregator or SkipTagsAggregator([("spell(", ")")]),
+            **kwargs,
+        )
+
+        # Store service configuration
+        self._api_key = api_key
+        self._url = url
+        self._voice_id = voice_id
+        self._model = model
+        self._settings = {
+            "speaker": voice_id,
+            "modelId": model,
+            "audioFormat": "pcm",
+            "samplingRate": 0,
+            "lang": self.language_to_service_language(params.language)
+            if params.language
+            else "eng",
+            "speedAlpha": params.speed_alpha,
+            "reduceLatency": params.reduce_latency,
+        }
+
+        # State tracking
+        self._context_id = None  # Tracks current turn
+        self._receive_task = None
+        self._cumulative_time = 0  # Accumulates time across messages
+
+    def can_generate_metrics(self) -> bool:
+        return True
+
+    def language_to_service_language(self, language: Language) -> str | None:
+        """Convert pipecat language to Rime language code."""
+        return language_to_rime_language(language)
+
+    async def set_model(self, model: str):
+        """Update the TTS model."""
+        self._model = model
+        await super().set_model(model)
+
+    def _build_msg(self, text: str = "") -> dict:
+        """Build JSON message for Rime API."""
+        return {"text": text, "contextId": self._context_id}
+
+    def _build_clear_msg(self) -> dict:
+        """Build clear operation message."""
+        return {"operation": "clear"}
+
+    def _build_eos_msg(self) -> dict:
+        """Build end-of-stream operation message."""
+        return {"operation": "eos"}
+
+    async def start(self, frame: StartFrame):
+        """Start the service and establish websocket connection."""
+        await super().start(frame)
+        self._settings["samplingRate"] = self.sample_rate
+        await self._connect()
+
+    async def stop(self, frame: EndFrame):
+        """Stop the service and close connection."""
+        await super().stop(frame)
+        await self._disconnect()
+
+    async def cancel(self, frame: CancelFrame):
+        """Cancel current operation and clean up."""
+        await super().cancel(frame)
+        await self._disconnect()
+
+    async def _connect(self):
+        """Establish websocket connection and start receive task."""
+        await self._connect_websocket()
+
+        if not self._receive_task:
+            self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
+
+    async def _disconnect(self):
+        """Close websocket connection and clean up tasks."""
+        if self._receive_task:
+            await self.cancel_task(self._receive_task)
+            self._receive_task = None
+
+        await self._disconnect_websocket()
+
+    async def _connect_websocket(self):
+        """Connect to Rime websocket API with configured settings."""
+        try:
+            if self._websocket:
+                return
+
+            params = "&".join(f"{k}={v}" for k, v in self._settings.items())
+            url = f"{self._url}?{params}"
+            headers = {"Authorization": f"Bearer {self._api_key}"}
+            self._websocket = await websockets.connect(url, extra_headers=headers)
+        except Exception as e:
+            logger.error(f"{self} initialization error: {e}")
+            self._websocket = None
+            await self._call_event_handler("on_connection_error", f"{e}")
+
+    async def _disconnect_websocket(self):
+        """Close websocket connection and reset state."""
+        try:
+            await self.stop_all_metrics()
+            if self._websocket:
+                await self._websocket.send(json.dumps(self._build_eos_msg()))
+                await self._websocket.close()
+                self._websocket = None
+            self._context_id = None
+        except Exception as e:
+            logger.error(f"{self} error closing websocket: {e}")
+
+    def _get_websocket(self):
+        """Get active websocket connection or raise exception."""
+        if self._websocket:
+            return self._websocket
+        raise Exception("Websocket not connected")
+
+    async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
+        """Handle interruption by clearing current context."""
+        await super()._handle_interruption(frame, direction)
+        await self.stop_all_metrics()
+        if self._context_id:
+            await self._get_websocket().send(json.dumps(self._build_clear_msg()))
+            self._context_id = None
+
+    def _calculate_word_times(self, words: list, starts: list, ends: list) -> list:
+        """Calculate word timing pairs with proper spacing and punctuation.
+
+        Args:
+            words: List of words from Rime.
+            starts: List of start times for each word.
+            ends: List of end times for each word.
+
+        Returns:
+            List of (word, timestamp) pairs with proper timing.
+        """
+        word_pairs = []
+        for i, (word, start_time, _) in enumerate(zip(words, starts, ends)):
+            if not word.strip():
+                continue
+
+            # Adjust timing by adding cumulative time
+            adjusted_start = start_time + self._cumulative_time
+
+            # Handle punctuation by appending to previous word
+            is_punctuation = bool(word.strip(",.!?") == "")
+            if is_punctuation and word_pairs:
+                prev_word, prev_time = word_pairs[-1]
+                word_pairs[-1] = (prev_word + word, prev_time)
+            else:
+                word_pairs.append((word, adjusted_start))
+
+        return word_pairs
+
+    async def flush_audio(self):
+        if not self._context_id or not self._websocket:
+            return
+
+        logger.trace(f"{self}: flushing audio")
+        await self._get_websocket().send(json.dumps({"text": " "}))
+        self._context_id = None
+
+    async def _receive_messages(self):
+        """Process incoming websocket messages."""
+        async for message in self._get_websocket():
+            msg = json.loads(message)
+
+            if not msg or not self.audio_context_available(msg["contextId"]):
+                continue
+
+            if msg["type"] == "chunk":
+                # Process audio chunk
+                await self.stop_ttfb_metrics()
+                self.start_word_timestamps()
+                frame = TTSAudioRawFrame(
+                    audio=base64.b64decode(msg["data"]),
+                    sample_rate=self.sample_rate,
+                    num_channels=1,
+                )
+                await self.append_to_audio_context(msg["contextId"], frame)
+
+            elif msg["type"] == "timestamps":
+                # Process word timing information
+                timestamps = msg.get("word_timestamps", {})
+                words = timestamps.get("words", [])
+                starts = timestamps.get("start", [])
+                ends = timestamps.get("end", [])
+
+                if words and starts:
+                    # Calculate word timing pairs
+                    word_pairs = self._calculate_word_times(words, starts, ends)
+                    if word_pairs:
+                        await self.add_word_timestamps(word_pairs)
+                        self._cumulative_time = ends[-1] + self._cumulative_time
+                        logger.debug(f"Updated cumulative time to: {self._cumulative_time}")
+
+            elif msg["type"] == "error":
+                logger.error(f"{self} error: {msg}")
+                await self.push_frame(TTSStoppedFrame())
+                await self.stop_all_metrics()
+                await self.push_error(ErrorFrame(f"{self} error: {msg['message']}"))
+                self._context_id = None
+
+    async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
+        """Push frame and handle end-of-turn conditions."""
+        await super().push_frame(frame, direction)
+        if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
+            if isinstance(frame, TTSStoppedFrame):
+                await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)])
+
+    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+        """Generate speech from text.
+
+        Args:
+            text: The text to convert to speech.
+
+        Yields:
+            Frames containing audio data and timing information.
+        """
+        logger.debug(f"{self}: Generating TTS [{text}]")
+        try:
+            if not self._websocket:
+                await self._connect()
+
+            try:
+                if not self._context_id:
+                    await self.start_ttfb_metrics()
+                    yield TTSStartedFrame()
+                    self._cumulative_time = 0
+                    self._context_id = str(uuid.uuid4())
+                    await self.create_audio_context(self._context_id)
+
+                msg = self._build_msg(text=text)
+                await self._get_websocket().send(json.dumps(msg))
+                await self.start_tts_usage_metrics(text)
+            except Exception as e:
+                logger.error(f"{self} error sending message: {e}")
+                yield TTSStoppedFrame()
+                await self._disconnect()
+                await self._connect()
+                return
+            yield None
+        except Exception as e:
+            logger.error(f"{self} exception: {e}")
 
 
 class RimeHttpTTSService(TTSService):
@@ -32,18 +355,19 @@ class RimeHttpTTSService(TTSService):
         self,
         *,
         api_key: str,
-        voice_id: str = "eva",
-        model: str = "mist",
-        sample_rate: int = 24000,
+        voice_id: str,
+        aiohttp_session: aiohttp.ClientSession,
+        model: str = "mistv2",
+        sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
     ):
         super().__init__(sample_rate=sample_rate, **kwargs)
 
         self._api_key = api_key
+        self._session = aiohttp_session
         self._base_url = "https://users.rime.ai/v1/rime-tts"
         self._settings = {
-            "samplingRate": sample_rate,
             "speedAlpha": params.speed_alpha,
             "reduceLatency": params.reduce_latency,
             "pauseBetweenBrackets": params.pause_between_brackets,
@@ -59,7 +383,7 @@ class RimeHttpTTSService(TTSService):
         return True
 
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         headers = {
             "Accept": "audio/pcm",
@@ -71,39 +395,35 @@ class RimeHttpTTSService(TTSService):
         payload["text"] = text
         payload["speaker"] = self._voice_id
         payload["modelId"] = self._model_name
+        payload["samplingRate"] = self.sample_rate
 
         try:
             await self.start_ttfb_metrics()
-            await self.start_tts_usage_metrics(text)
 
-            yield TTSStartedFrame()
+            async with self._session.post(
+                self._base_url, json=payload, headers=headers
+            ) as response:
+                if response.status != 200:
+                    error_message = f"Rime TTS error: HTTP {response.status}"
+                    logger.error(error_message)
+                    yield ErrorFrame(error=error_message)
+                    return
 
-            async with aiohttp.ClientSession() as session:
-                async with session.post(self._base_url, json=payload, headers=headers) as response:
-                    if response.status != 200:
-                        error_message = f"Rime TTS error: HTTP {response.status}"
-                        logger.error(error_message)
-                        yield ErrorFrame(error=error_message)
-                        return
+                await self.start_tts_usage_metrics(text)
 
-                    # Process the streaming response
-                    chunk_size = 8192
-                    first_chunk = True
+                yield TTSStartedFrame()
 
-                    async for chunk in response.content.iter_chunked(chunk_size):
-                        if first_chunk:
-                            await self.stop_ttfb_metrics()
-                            first_chunk = False
-
-                        if chunk:
-                            frame = TTSAudioRawFrame(chunk, self._settings["samplingRate"], 1)
-                            yield frame
-
-            yield TTSStoppedFrame()
+                # Process the streaming response
+                CHUNK_SIZE = 1024
 
+                async for chunk in response.content.iter_chunked(CHUNK_SIZE):
+                    if len(chunk) > 0:
+                        await self.stop_ttfb_metrics()
+                        frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
+                        yield frame
         except Exception as e:
             logger.exception(f"Error generating TTS: {e}")
             yield ErrorFrame(error=f"Rime TTS error: {str(e)}")
-
         finally:
+            await self.stop_ttfb_metrics()
             yield TTSStoppedFrame()

src/pipecat/services/riva.py

@@ -49,7 +49,7 @@ class FastPitchTTSService(TTSService):
         api_key: str,
         server: str = "grpc.nvcf.nvidia.com:443",
         voice_id: str = "English-US.Female-1",
-        sample_rate: int = 24000,
+        sample_rate: Optional[int] = None,
         function_id: str = "0149dedb-2be8-4195-b9a0-e57e0e14f972",
         params: InputParams = InputParams(),
         **kwargs,
@@ -57,7 +57,6 @@ class FastPitchTTSService(TTSService):
         super().__init__(sample_rate=sample_rate, **kwargs)
         self._api_key = api_key
         self._voice_id = voice_id
-        self._sample_rate = sample_rate
         self._language_code = params.language
         self._quality = params.quality
 
@@ -87,7 +86,7 @@ class FastPitchTTSService(TTSService):
                     text,
                     self._voice_id,
                     self._language_code,
-                    sample_rate_hz=self._sample_rate,
+                    sample_rate_hz=self.sample_rate,
                     audio_prompt_file=None,
                     quality=self._quality,
                     custom_dictionary={},
@@ -102,7 +101,7 @@ class FastPitchTTSService(TTSService):
         await self.start_ttfb_metrics()
         yield TTSStartedFrame()
 
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
             queue = asyncio.Queue()
@@ -114,7 +113,7 @@ class FastPitchTTSService(TTSService):
                 await self.stop_ttfb_metrics()
                 frame = TTSAudioRawFrame(
                     audio=resp.audio,
-                    sample_rate=self._sample_rate,
+                    sample_rate=self.sample_rate,
                     num_channels=1,
                 )
                 yield frame
@@ -136,10 +135,11 @@ class ParakeetSTTService(STTService):
         api_key: str,
         server: str = "grpc.nvcf.nvidia.com:443",
         function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081",
+        sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
     ):
-        super().__init__(**kwargs)
+        super().__init__(sample_rate=sample_rate, **kwargs)
         self._api_key = api_key
         self._profanity_filter = False
         self._automatic_punctuation = False
@@ -154,7 +154,6 @@ class ParakeetSTTService(STTService):
         self._stop_history_eou = -1
         self._stop_threshold_eou = -1.0
         self._custom_configuration = ""
-        self._sample_rate: int = 16000
 
         self.set_model_name("parakeet-ctc-1.1b-asr")
 
@@ -166,6 +165,20 @@ class ParakeetSTTService(STTService):
 
         self._asr_service = riva.client.ASRService(auth)
 
+        self._queue = asyncio.Queue()
+        self._config = None
+        self._thread_task = None
+        self._response_task = None
+
+    def can_generate_metrics(self) -> bool:
+        return False
+
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+
+        if self._config:
+            return
+
         config = riva.client.StreamingRecognitionConfig(
             config=riva.client.RecognitionConfig(
                 encoding=riva.client.AudioEncoding.LINEAR_PCM,
@@ -175,14 +188,16 @@ class ParakeetSTTService(STTService):
                 profanity_filter=self._profanity_filter,
                 enable_automatic_punctuation=self._automatic_punctuation,
                 verbatim_transcripts=not self._no_verbatim_transcripts,
-                sample_rate_hertz=self._sample_rate,
+                sample_rate_hertz=self.sample_rate,
                 audio_channel_count=1,
             ),
             interim_results=True,
         )
+
         riva.client.add_word_boosting_to_config(
             config, self._boosted_lm_words, self._boosted_lm_score
         )
+
         riva.client.add_endpoint_parameters_to_config(
             config,
             self._start_history,
@@ -193,18 +208,15 @@ class ParakeetSTTService(STTService):
             self._stop_threshold_eou,
         )
         riva.client.add_custom_configuration_to_config(config, self._custom_configuration)
+
         self._config = config
 
-        self._queue = asyncio.Queue()
+        if not self._thread_task:
+            self._thread_task = self.create_task(self._thread_task_handler())
 
-    def can_generate_metrics(self) -> bool:
-        return False
-
-    async def start(self, frame: StartFrame):
-        await super().start(frame)
-        self._thread_task = self.create_task(self._thread_task_handler())
-        self._response_task = self.create_task(self._response_task_handler())
-        self._response_queue = asyncio.Queue()
+        if not self._response_task:
+            self._response_queue = asyncio.Queue()
+            self._response_task = self.create_task(self._response_task_handler())
 
     async def stop(self, frame: EndFrame):
         await super().stop(frame)
@@ -215,8 +227,13 @@ class ParakeetSTTService(STTService):
         await self._stop_tasks()
 
     async def _stop_tasks(self):
-        await self.cancel_task(self._thread_task)
-        await self.cancel_task(self._response_task)
+        if self._thread_task:
+            await self.cancel_task(self._thread_task)
+            self._thread_task = None
+
+        if self._response_task:
+            await self.cancel_task(self._response_task)
+            self._response_task = None
 
     def _response_handler(self):
         responses = self._asr_service.streaming_response_generator(

src/pipecat/services/simli.py

@@ -41,16 +41,23 @@ class SimliVideoService(FrameProcessor):
 
         self._pipecat_resampler_event = asyncio.Event()
         self._pipecat_resampler: AudioResampler = None
-        self._simli_resampler = AudioResampler("s16", 1, 16000)
+        self._simli_resampler = AudioResampler("s16", "mono", 16000)
 
+        self._initialized = False
         self._audio_task: asyncio.Task = None
         self._video_task: asyncio.Task = None
 
     async def _start_connection(self):
-        await self._simli_client.Initialize()
+        if not self._initialized:
+            await self._simli_client.Initialize()
+            self._initialized = True
+
         # Create task to consume and process audio and video
-        self._audio_task = self.create_task(self._consume_and_process_audio())
-        self._video_task = self.create_task(self._consume_and_process_video())
+        if not self._audio_task:
+            self._audio_task = self.create_task(self._consume_and_process_audio())
+
+        if not self._video_task:
+            self._video_task = self.create_task(self._consume_and_process_video())
 
     async def _consume_and_process_audio(self):
         await self._pipecat_resampler_event.wait()
@@ -117,5 +124,7 @@ class SimliVideoService(FrameProcessor):
         await self._simli_client.stop()
         if self._audio_task:
             await self.cancel_task(self._audio_task)
+            self._audio_task = None
         if self._video_task:
             await self.cancel_task(self._video_task)
+            self._video_task = None

src/pipecat/services/tavus.py

@@ -37,6 +37,7 @@ class TavusVideoService(AIService):
         replica_id: str,
         persona_id: str = "pipecat0",  # Use `pipecat0` so that your TTS voice is used in place of the Tavus persona
         session: aiohttp.ClientSession,
+        sample_rate: int = 16000,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -44,6 +45,7 @@ class TavusVideoService(AIService):
         self._replica_id = replica_id
         self._persona_id = persona_id
         self._session = session
+        self._sample_rate = sample_rate
 
         self._conversation_id: str
 
@@ -94,7 +96,7 @@ class TavusVideoService(AIService):
     async def _encode_audio_and_send(self, audio: bytes, in_rate: int, done: bool) -> None:
         """Encodes audio to base64 and sends it to Tavus"""
         if not done:
-            audio = await self._resampler.resample(audio, in_rate, 16000)
+            audio = await self._resampler.resample(audio, in_rate, self._sample_rate)
         audio_base64 = base64.b64encode(audio).decode("utf-8")
         logger.trace(f"{self}: sending {len(audio)} bytes")
         await self._send_audio_message(audio_base64, done=done)
@@ -108,7 +110,7 @@ class TavusVideoService(AIService):
         elif isinstance(frame, TTSAudioRawFrame):
             await self._encode_audio_and_send(frame.audio, frame.sample_rate, done=False)
         elif isinstance(frame, TTSStoppedFrame):
-            await self._encode_audio_and_send(b"\x00", 16000, done=True)
+            await self._encode_audio_and_send(b"\x00", self._sample_rate, done=True)
             await self.stop_ttfb_metrics()
             await self.stop_processing_metrics()
         elif isinstance(frame, StartInterruptionFrame):
@@ -137,6 +139,7 @@ class TavusVideoService(AIService):
                     "inference_id": self._current_idx_str,
                     "audio": audio_base64,
                     "done": done,
+                    "sample_rate": self._sample_rate,
                 },
             }
         )

src/pipecat/services/ultravox.py

@@ -0,0 +1,403 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""This module implements Ultravox speech-to-text with a locally-loaded model."""
+
+import json
+import os
+import time
+from typing import AsyncGenerator, List, Optional
+
+import numpy as np
+from huggingface_hub import login
+from loguru import logger
+
+from pipecat.frames.frames import (
+    AudioRawFrame,
+    CancelFrame,
+    EndFrame,
+    ErrorFrame,
+    Frame,
+    LLMFullResponseEndFrame,
+    LLMFullResponseStartFrame,
+    LLMTextFrame,
+    StartFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.ai_services import AIService
+
+try:
+    from transformers import AutoTokenizer
+    from vllm import AsyncLLMEngine, SamplingParams
+    from vllm.engine.arg_utils import AsyncEngineArgs
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error("In order to use Ultravox, you need to `pip install pipecat-ai[ultravox]`.")
+    raise Exception(f"Missing module: {e}")
+
+
+class AudioBuffer:
+    """Buffer to collect audio frames before processing.
+
+    Attributes:
+        frames: List of AudioRawFrames to process
+        started_at: Timestamp when speech started
+        is_processing: Flag to prevent concurrent processing
+    """
+
+    def __init__(self):
+        self.frames: List[AudioRawFrame] = []
+        self.started_at: Optional[float] = None
+        self.is_processing: bool = False
+
+
+class UltravoxModel:
+    """Model wrapper for the Ultravox multimodal model.
+
+    This class handles loading and running the Ultravox model for speech-to-text.
+
+    Args:
+        model_name: The name or path of the Ultravox model to load
+
+    Attributes:
+        model_name: The name of the loaded model
+        engine: The vLLM engine for model inference
+        tokenizer: The tokenizer for the model
+        stop_token_ids: Optional token IDs to stop generation
+    """
+
+    def __init__(self, model_name: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b"):
+        self.model_name = model_name
+        self._initialize_engine()
+        self._initialize_tokenizer()
+        self.stop_token_ids = None
+
+    def _initialize_engine(self):
+        """Initialize the vLLM engine for inference."""
+        engine_args = AsyncEngineArgs(
+            model=self.model_name,
+            gpu_memory_utilization=0.9,
+            max_model_len=8192,
+            trust_remote_code=True,
+        )
+        self.engine = AsyncLLMEngine.from_engine_args(engine_args)
+
+    def _initialize_tokenizer(self):
+        """Initialize the tokenizer for the model."""
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+
+    def format_prompt(self, messages: list):
+        """Format chat messages into a prompt for the model.
+
+        Args:
+            messages: List of message dictionaries with 'role' and 'content'
+
+        Returns:
+            str: Formatted prompt string
+        """
+        return self.tokenizer.apply_chat_template(
+            messages, tokenize=False, add_generation_prompt=True
+        )
+
+    async def generate(
+        self,
+        messages: list,
+        temperature: float = 0.7,
+        max_tokens: int = 100,
+        audio: np.ndarray = None,
+    ):
+        """Generate text from audio input using the model.
+
+        Args:
+            messages: List of message dictionaries
+            temperature: Sampling temperature
+            max_tokens: Maximum tokens to generate
+            audio: Audio data as numpy array
+
+        Yields:
+            str: JSON chunks of the generated response
+        """
+        sampling_params = SamplingParams(
+            temperature=temperature, max_tokens=max_tokens, stop_token_ids=self.stop_token_ids
+        )
+
+        mm_data = {"audio": audio}
+        inputs = {"prompt": self.format_prompt(messages), "multi_modal_data": mm_data}
+        results_generator = self.engine.generate(inputs, sampling_params, str(time.time()))
+
+        previous_text = ""
+        first_chunk = True
+
+        async for output in results_generator:
+            prompt_output = output.outputs
+            new_text = prompt_output[0].text[len(previous_text) :]
+            previous_text = prompt_output[0].text
+
+            # Construct OpenAI-compatible chunk
+            chunk = {
+                "id": str(int(time.time() * 1000)),
+                "object": "chat.completion.chunk",
+                "created": int(time.time()),
+                "model": self.model_name,
+                "choices": [
+                    {
+                        "index": 0,
+                        "delta": {},
+                        "finish_reason": None,
+                    }
+                ],
+            }
+
+            # Include the role in the first chunk
+            if first_chunk:
+                chunk["choices"][0]["delta"]["role"] = "assistant"
+                first_chunk = False
+
+            # Add new text to the delta if any
+            if new_text:
+                chunk["choices"][0]["delta"]["content"] = new_text
+
+            # Capture a finish reason if it's provided
+            finish_reason = prompt_output[0].finish_reason or None
+            if finish_reason and finish_reason != "none":
+                chunk["choices"][0]["finish_reason"] = finish_reason
+
+            yield json.dumps(chunk)
+
+
+class UltravoxSTTService(AIService):
+    """Service to transcribe audio using the Ultravox multimodal model.
+
+    This service collects audio frames and processes them with Ultravox
+    to generate text transcriptions.
+
+    Args:
+        model_size: The Ultravox model to use (ModelSize enum or string)
+        hf_token: Hugging Face token for model access
+        temperature: Sampling temperature for generation
+        max_tokens: Maximum tokens to generate
+        **kwargs: Additional arguments passed to AIService
+
+    Attributes:
+        model: The UltravoxModel instance
+        buffer: Buffer to collect audio frames
+        temperature: Temperature for text generation
+        max_tokens: Maximum tokens to generate
+        _connection_active: Flag indicating if service is active
+    """
+
+    def __init__(
+        self,
+        *,
+        model_size: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b",
+        hf_token: Optional[str] = None,
+        temperature: float = 0.7,
+        max_tokens: int = 100,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        # Authenticate with Hugging Face if token provided
+        if hf_token:
+            login(token=hf_token)
+        elif os.environ.get("HF_TOKEN"):
+            login(token=os.environ.get("HF_TOKEN"))
+        else:
+            logger.warning("No Hugging Face token provided. Model may not load correctly.")
+
+        # Initialize model
+        model_name = model_size if isinstance(model_size, str) else model_size.value
+        self._model = UltravoxModel(model_name=model_name)
+
+        # Initialize service state
+        self._buffer = AudioBuffer()
+        self._temperature = temperature
+        self._max_tokens = max_tokens
+        self._connection_active = False
+
+        logger.info(f"Initialized UltravoxSTTService with model: {model_name}")
+
+    def can_generate_metrics(self) -> bool:
+        """Indicates whether this service can generate metrics.
+
+        Returns:
+            bool: True, as this service supports metric generation.
+        """
+        return True
+
+    async def start(self, frame: StartFrame):
+        """Handle service start.
+
+        Args:
+            frame: StartFrame that triggered this method
+        """
+        await super().start(frame)
+        self._connection_active = True
+        logger.info("UltravoxSTTService started")
+
+    async def stop(self, frame: EndFrame):
+        """Handle service stop.
+
+        Args:
+            frame: EndFrame that triggered this method
+        """
+        await super().stop(frame)
+        self._connection_active = False
+        logger.info("UltravoxSTTService stopped")
+
+    async def cancel(self, frame: CancelFrame):
+        """Handle service cancellation.
+
+        Args:
+            frame: CancelFrame that triggered this method
+        """
+        await super().cancel(frame)
+        self._connection_active = False
+        self._buffer = AudioBuffer()
+        logger.info("UltravoxSTTService cancelled")
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames.
+
+        This method collects audio frames and processes them when speech ends.
+
+        Args:
+            frame: The frame to process
+            direction: Direction of the frame (input/output)
+        """
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, UserStartedSpeakingFrame):
+            logger.info("Speech started")
+            self._buffer = AudioBuffer()
+            self._buffer.started_at = time.time()
+
+        elif isinstance(frame, AudioRawFrame) and self._buffer.started_at is not None:
+            self._buffer.frames.append(frame)
+
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            if self._buffer.frames and not self._buffer.is_processing:
+                logger.info("Speech ended, processing buffer...")
+                await self.process_generator(self._process_audio_buffer())
+                return  # Return early to avoid pushing None frame
+
+        # Only push the original frame if we haven't processed audio
+        if frame is not None:
+            await self.push_frame(frame, direction)
+
+    async def _process_audio_buffer(self) -> AsyncGenerator[Frame, None]:
+        """Process collected audio frames with Ultravox.
+
+        This method concatenates audio frames, processes them with the model,
+        and yields the resulting text frames.
+
+        Yields:
+            Frame: TextFrame containing the transcribed text
+        """
+        try:
+            self._buffer.is_processing = True
+
+            # Check if we have valid frames before processing
+            if not self._buffer.frames:
+                logger.warning("No audio frames to process")
+                yield ErrorFrame("No audio frames to process")
+                return
+
+            # Process audio frames
+            audio_arrays = []
+            for f in self._buffer.frames:
+                if hasattr(f, "audio") and f.audio:
+                    # Handle bytes data - these are int16 PCM samples
+                    if isinstance(f.audio, bytes):
+                        try:
+                            # Convert bytes to int16 array
+                            arr = np.frombuffer(f.audio, dtype=np.int16)
+                            if arr.size > 0:  # Check if array is not empty
+                                audio_arrays.append(arr)
+                        except Exception as e:
+                            logger.error(f"Error processing bytes audio frame: {e}")
+                    # Handle numpy array data
+                    elif isinstance(f.audio, np.ndarray):
+                        if f.audio.size > 0:  # Check if array is not empty
+                            # Ensure it's int16 data
+                            if f.audio.dtype != np.int16:
+                                logger.info(f"Converting array from {f.audio.dtype} to int16")
+                                audio_arrays.append(f.audio.astype(np.int16))
+                            else:
+                                audio_arrays.append(f.audio)
+
+            # Only proceed if we have valid audio arrays
+            if not audio_arrays:
+                logger.warning("No valid audio data found in frames")
+                yield ErrorFrame("No valid audio data found in frames")
+                return
+
+            # Concatenate audio frames - all should be int16 now
+            audio_data = np.concatenate(audio_arrays)
+
+            audio_int16 = audio_data  # Already in int16 format
+            # Save int16 audio
+
+            # Convert int16 to float32 and normalize for model input
+            audio_float32 = audio_int16.astype(np.float32) / 32768.0
+
+            # Generate text using the model
+            if self._model:
+                try:
+                    logger.info("Generating text from audio using model...")
+                    full_response = ""
+
+                    # Start metrics tracking
+                    await self.start_ttfb_metrics()
+                    await self.start_processing_metrics()
+
+                    async for response in self.model.generate(
+                        messages=[{"role": "user", "content": "<|audio|>\n"}],
+                        temperature=self.temperature,
+                        max_tokens=self.max_tokens,
+                        audio=audio_float32,
+                    ):
+                        # Stop TTFB metrics after first response
+                        await self.stop_ttfb_metrics()
+
+                        chunk = json.loads(response)
+                        if "choices" in chunk and len(chunk["choices"]) > 0:
+                            delta = chunk["choices"][0]["delta"]
+                            if "content" in delta:
+                                new_text = delta["content"]
+                                full_response += new_text
+
+                    # Stop processing metrics after completion
+                    await self.stop_processing_metrics()
+
+                    logger.info(f"Generated text: {full_response}")
+                    # Create a transcription frame with the generated text
+                    yield LLMFullResponseStartFrame()
+
+                    text_frame = LLMTextFrame(text=full_response.strip())
+                    yield text_frame
+
+                    yield LLMFullResponseEndFrame()
+
+                except Exception as e:
+                    logger.error(f"Error generating text from model: {e}")
+                    yield ErrorFrame(f"Error generating text: {str(e)}")
+            else:
+                logger.warning("No model available for text generation")
+                yield ErrorFrame("No model available for text generation")
+
+        except Exception as e:
+            logger.error(f"Error processing audio buffer: {e}")
+            import traceback
+
+            logger.error(traceback.format_exc())
+            yield ErrorFrame(f"Error processing audio: {str(e)}")
+        finally:
+            self._buffer.is_processing = False
+            self._buffer.frames = []
+            self._buffer.started_at = None

src/pipecat/services/websocket_service.py

@@ -10,16 +10,19 @@ from typing import Awaitable, Callable, Optional
 
 import websockets
 from loguru import logger
+from websockets.protocol import State
 
 from pipecat.frames.frames import ErrorFrame
+from pipecat.utils.network import exponential_backoff_time
 
 
 class WebsocketService(ABC):
     """Base class for websocket-based services with reconnection logic."""
 
-    def __init__(self):
+    def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
         """Initialize websocket attributes."""
         self._websocket: Optional[websockets.WebSocketClientProtocol] = None
+        self._reconnect_on_error = reconnect_on_error
 
     async def _verify_connection(self) -> bool:
         """Verify websocket connection is working.
@@ -50,27 +53,6 @@ class WebsocketService(ABC):
         await self._connect_websocket()
         return await self._verify_connection()
 
-    def _calculate_wait_time(
-        self, attempt: int, min_wait: float = 4, max_wait: float = 10, multiplier: float = 1
-    ) -> float:
-        """Calculate exponential backoff wait time.
-
-        Args:
-            attempt: Current attempt number (1-based)
-            min_wait: Minimum wait time in seconds
-            max_wait: Maximum wait time in seconds
-            multiplier: Base multiplier for exponential calculation
-
-        Returns:
-            Wait time in seconds
-        """
-        try:
-            exp = 2 ** (attempt - 1) * multiplier
-            result = max(0, min(exp, max_wait))
-            return max(min_wait, result)
-        except (ValueError, ArithmeticError):
-            return max_wait
-
     async def _receive_task_handler(self, report_error: Callable[[ErrorFrame], Awaitable[None]]):
         """Handles WebSocket message receiving with automatic retry logic.
 
@@ -83,35 +65,63 @@ class WebsocketService(ABC):
         while True:
             try:
                 await self._receive_messages()
-                logger.debug(f"{self} connection established successfully")
                 retry_count = 0  # Reset counter on successful message receive
+                if self._websocket and self._websocket.state == State.CLOSED:
+                    raise websockets.ConnectionClosedOK(
+                        self._websocket.close_rcvd,
+                        self._websocket.close_sent,
+                        self._websocket.close_rcvd_then_sent,
+                    )
             except Exception as e:
-                retry_count += 1
-                if retry_count >= MAX_RETRIES:
-                    message = f"{self} error receiving messages: {e}"
-                    logger.error(message)
-                    await report_error(ErrorFrame(message, fatal=True))
+                message = f"{self} error receiving messages: {e}"
+                logger.error(message)
+
+                if self._reconnect_on_error:
+                    retry_count += 1
+                    if retry_count >= MAX_RETRIES:
+                        await report_error(ErrorFrame(message, fatal=True))
+                        break
+
+                    logger.warning(f"{self} connection error, will retry: {e}")
+                    await report_error(ErrorFrame(message))
+
+                    try:
+                        if await self._reconnect_websocket(retry_count):
+                            retry_count = 0  # Reset counter on successful reconnection
+                        wait_time = exponential_backoff_time(retry_count)
+                        await asyncio.sleep(wait_time)
+                    except Exception as reconnect_error:
+                        logger.error(f"{self} reconnection failed: {reconnect_error}")
+                else:
+                    await report_error(ErrorFrame(message))
                     break
 
-                logger.warning(f"{self} connection error, will retry: {e}")
+    @abstractmethod
+    async def _connect(self):
+        """Implement service-specific connection logic. This function will
+        connect to the websocket via _connect_websocket() among other connection
+        logic."""
+        pass
 
-                try:
-                    if await self._reconnect_websocket(retry_count):
-                        retry_count = 0  # Reset counter on successful reconnection
-                    wait_time = self._calculate_wait_time(retry_count)
-                    await asyncio.sleep(wait_time)
-                except Exception as reconnect_error:
-                    logger.error(f"{self} reconnection failed: {reconnect_error}")
-                    continue
+    @abstractmethod
+    async def _disconnect(self):
+        """Implement service-specific disconnection logic. This function will
+        disconnect to the websocket via _connect_websocket() among other
+        connection logic.
+
+        """
+        pass
 
     @abstractmethod
     async def _connect_websocket(self):
-        """Implement service-specific websocket connection logic."""
+        """Implement service-specific websocket connection logic. This function
+        should only connect to the websocket."""
         pass
 
     @abstractmethod
     async def _disconnect_websocket(self):
-        """Implement service-specific websocket disconnection logic."""
+        """Implement service-specific websocket disconnection logic. This
+        function should only disconnect from the websocket."""
         pass
 
     @abstractmethod

src/pipecat/services/whisper.py

@@ -8,13 +8,14 @@
 
 import asyncio
 from enum import Enum
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 
 import numpy as np
 from loguru import logger
 
 from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
 from pipecat.services.ai_services import SegmentedSTTService
+from pipecat.transcriptions.language import Language
 from pipecat.utils.time import time_now_iso8601
 
 try:
@@ -26,18 +27,219 @@ except ModuleNotFoundError as e:
 
 
 class Model(Enum):
-    """Class of basic Whisper model selection options"""
+    """Class of basic Whisper model selection options.
 
+    Available models:
+        Multilingual models:
+            TINY: Smallest multilingual model
+            BASE: Basic multilingual model
+            MEDIUM: Good balance for multilingual
+            LARGE: Best quality multilingual
+            DISTIL_LARGE_V2: Fast multilingual
+
+        English-only models:
+            DISTIL_MEDIUM_EN: Fast English-only
+    """
+
+    # Multilingual models
     TINY = "tiny"
     BASE = "base"
     MEDIUM = "medium"
     LARGE = "large-v3"
     DISTIL_LARGE_V2 = "Systran/faster-distil-whisper-large-v2"
+
+    # English-only models
     DISTIL_MEDIUM_EN = "Systran/faster-distil-whisper-medium.en"
 
 
+def language_to_whisper_language(language: Language) -> Optional[str]:
+    """Maps pipecat Language enum to Whisper language codes.
+
+    Args:
+        language: A Language enum value representing the input language.
+
+    Returns:
+        str or None: The corresponding Whisper language code, or None if not supported.
+
+    Note:
+        Only includes languages officially supported by Whisper.
+    """
+    language_map = {
+        # Arabic
+        Language.AR: "ar",
+        Language.AR_AE: "ar",
+        Language.AR_BH: "ar",
+        Language.AR_DZ: "ar",
+        Language.AR_EG: "ar",
+        Language.AR_IQ: "ar",
+        Language.AR_JO: "ar",
+        Language.AR_KW: "ar",
+        Language.AR_LB: "ar",
+        Language.AR_LY: "ar",
+        Language.AR_MA: "ar",
+        Language.AR_OM: "ar",
+        Language.AR_QA: "ar",
+        Language.AR_SA: "ar",
+        Language.AR_SY: "ar",
+        Language.AR_TN: "ar",
+        Language.AR_YE: "ar",
+        # Bengali
+        Language.BN: "bn",
+        Language.BN_BD: "bn",
+        Language.BN_IN: "bn",
+        # Czech
+        Language.CS: "cs",
+        Language.CS_CZ: "cs",
+        # Danish
+        Language.DA: "da",
+        Language.DA_DK: "da",
+        # German
+        Language.DE: "de",
+        Language.DE_AT: "de",
+        Language.DE_CH: "de",
+        Language.DE_DE: "de",
+        # Greek
+        Language.EL: "el",
+        Language.EL_GR: "el",
+        # English
+        Language.EN: "en",
+        Language.EN_AU: "en",
+        Language.EN_CA: "en",
+        Language.EN_GB: "en",
+        Language.EN_HK: "en",
+        Language.EN_IE: "en",
+        Language.EN_IN: "en",
+        Language.EN_KE: "en",
+        Language.EN_NG: "en",
+        Language.EN_NZ: "en",
+        Language.EN_PH: "en",
+        Language.EN_SG: "en",
+        Language.EN_TZ: "en",
+        Language.EN_US: "en",
+        Language.EN_ZA: "en",
+        # Spanish
+        Language.ES: "es",
+        Language.ES_AR: "es",
+        Language.ES_BO: "es",
+        Language.ES_CL: "es",
+        Language.ES_CO: "es",
+        Language.ES_CR: "es",
+        Language.ES_CU: "es",
+        Language.ES_DO: "es",
+        Language.ES_EC: "es",
+        Language.ES_ES: "es",
+        Language.ES_GQ: "es",
+        Language.ES_GT: "es",
+        Language.ES_HN: "es",
+        Language.ES_MX: "es",
+        Language.ES_NI: "es",
+        Language.ES_PA: "es",
+        Language.ES_PE: "es",
+        Language.ES_PR: "es",
+        Language.ES_PY: "es",
+        Language.ES_SV: "es",
+        Language.ES_US: "es",
+        Language.ES_UY: "es",
+        Language.ES_VE: "es",
+        # Persian
+        Language.FA: "fa",
+        Language.FA_IR: "fa",
+        # Finnish
+        Language.FI: "fi",
+        Language.FI_FI: "fi",
+        # French
+        Language.FR: "fr",
+        Language.FR_BE: "fr",
+        Language.FR_CA: "fr",
+        Language.FR_CH: "fr",
+        Language.FR_FR: "fr",
+        # Hindi
+        Language.HI: "hi",
+        Language.HI_IN: "hi",
+        # Hungarian
+        Language.HU: "hu",
+        Language.HU_HU: "hu",
+        # Indonesian
+        Language.ID: "id",
+        Language.ID_ID: "id",
+        # Italian
+        Language.IT: "it",
+        Language.IT_IT: "it",
+        # Japanese
+        Language.JA: "ja",
+        Language.JA_JP: "ja",
+        # Korean
+        Language.KO: "ko",
+        Language.KO_KR: "ko",
+        # Dutch
+        Language.NL: "nl",
+        Language.NL_BE: "nl",
+        Language.NL_NL: "nl",
+        # Polish
+        Language.PL: "pl",
+        Language.PL_PL: "pl",
+        # Portuguese
+        Language.PT: "pt",
+        Language.PT_BR: "pt",
+        Language.PT_PT: "pt",
+        # Romanian
+        Language.RO: "ro",
+        Language.RO_RO: "ro",
+        # Russian
+        Language.RU: "ru",
+        Language.RU_RU: "ru",
+        # Slovak
+        Language.SK: "sk",
+        Language.SK_SK: "sk",
+        # Swedish
+        Language.SV: "sv",
+        Language.SV_SE: "sv",
+        # Thai
+        Language.TH: "th",
+        Language.TH_TH: "th",
+        # Turkish
+        Language.TR: "tr",
+        Language.TR_TR: "tr",
+        # Ukrainian
+        Language.UK: "uk",
+        Language.UK_UA: "uk",
+        # Urdu
+        Language.UR: "ur",
+        Language.UR_IN: "ur",
+        Language.UR_PK: "ur",
+        # Vietnamese
+        Language.VI: "vi",
+        Language.VI_VN: "vi",
+        # Chinese
+        Language.ZH: "zh",
+        Language.ZH_CN: "zh",
+        Language.ZH_HK: "zh",
+        Language.ZH_TW: "zh",
+    }
+    return language_map.get(language)
+
+
 class WhisperSTTService(SegmentedSTTService):
-    """Class to transcribe audio with a locally-downloaded Whisper model"""
+    """Class to transcribe audio with a locally-downloaded Whisper model.
+
+    This service uses Faster Whisper to perform speech-to-text transcription on audio
+    segments. It supports multiple languages and various model sizes.
+
+    Args:
+        model: The Whisper model to use for transcription. Can be a Model enum or string.
+        device: The device to run inference on ('cpu', 'cuda', or 'auto').
+        compute_type: The compute type for inference ('default', 'int8', 'int8_float16', etc.).
+        no_speech_prob: Probability threshold for filtering out non-speech segments.
+        language: The default language for transcription.
+        **kwargs: Additional arguments passed to SegmentedSTTService.
+
+    Attributes:
+        _device: The device used for inference.
+        _compute_type: The compute type for inference.
+        _no_speech_prob: Threshold for non-speech filtering.
+        _model: The loaded Whisper model instance.
+        _settings: Dictionary containing service settings.
+    """
 
     def __init__(
         self,
@@ -46,6 +248,7 @@ class WhisperSTTService(SegmentedSTTService):
         device: str = "auto",
         compute_type: str = "default",
         no_speech_prob: float = 0.4,
+        language: Language = Language.EN,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -53,15 +256,48 @@ class WhisperSTTService(SegmentedSTTService):
         self._compute_type = compute_type
         self.set_model_name(model if isinstance(model, str) else model.value)
         self._no_speech_prob = no_speech_prob
-        self._model: WhisperModel | None = None
+        self._model: Optional[WhisperModel] = None
+
+        self._settings = {
+            "language": language,
+        }
+
         self._load()
 
     def can_generate_metrics(self) -> bool:
+        """Indicates whether this service can generate metrics.
+
+        Returns:
+            bool: True, as this service supports metric generation.
+        """
         return True
 
+    def language_to_service_language(self, language: Language) -> Optional[str]:
+        """Convert from pipecat Language to Whisper language code.
+
+        Args:
+            language: The Language enum value to convert.
+
+        Returns:
+            str or None: The corresponding Whisper language code, or None if not supported.
+        """
+        return language_to_whisper_language(language)
+
+    async def set_language(self, language: Language):
+        """Set the language for transcription.
+
+        Args:
+            language: The Language enum value to use for transcription.
+        """
+        logger.info(f"Switching STT language to: [{language}]")
+        self._settings["language"] = language
+
     def _load(self):
-        """Loads the Whisper model. Note that if this is the first time
-        this model is being run, it will take time to download.
+        """Loads the Whisper model.
+
+        Note:
+            If this is the first time this model is being run,
+            it will take time to download from the Hugging Face model hub.
         """
         logger.debug("Loading Whisper model...")
         self._model = WhisperModel(
@@ -70,7 +306,19 @@ class WhisperSTTService(SegmentedSTTService):
         logger.debug("Loaded Whisper model")
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
-        """Transcribes given audio using Whisper"""
+        """Transcribes given audio using Whisper.
+
+        Args:
+            audio: Raw audio bytes in 16-bit PCM format.
+
+        Yields:
+            Frame: Either a TranscriptionFrame containing the transcribed text
+                  or an ErrorFrame if transcription fails.
+
+        Note:
+            The audio is expected to be 16-bit signed PCM data.
+            The service will normalize it to float32 in the range [-1, 1].
+        """
         if not self._model:
             logger.error(f"{self} error: Whisper model not available")
             yield ErrorFrame("Whisper model not available")
@@ -82,7 +330,10 @@ class WhisperSTTService(SegmentedSTTService):
         # Divide by 32768 because we have signed 16-bit data.
         audio_float = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0
 
-        segments, _ = await asyncio.to_thread(self._model.transcribe, audio_float)
+        whisper_lang = self.language_to_service_language(self._settings["language"])
+        segments, _ = await asyncio.to_thread(
+            self._model.transcribe, audio_float, language=whisper_lang
+        )
         text: str = ""
         for segment in segments:
             if segment.no_speech_prob < self._no_speech_prob:
@@ -93,4 +344,4 @@ class WhisperSTTService(SegmentedSTTService):
 
         if text:
             logger.debug(f"Transcription: [{text}]")
-            yield TranscriptionFrame(text, "", time_now_iso8601())
+            yield TranscriptionFrame(text, "", time_now_iso8601(), self._settings["language"])

src/pipecat/services/xtts.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-from typing import Any, AsyncGenerator, Dict
+from typing import Any, AsyncGenerator, Dict, Optional
 
 import aiohttp
 from loguru import logger
@@ -29,7 +29,7 @@ from pipecat.transcriptions.language import Language
 # https://github.com/coqui-ai/xtts-streaming-server
 
 
-def language_to_xtts_language(language: Language) -> str | None:
+def language_to_xtts_language(language: Language) -> Optional[str]:
     BASE_LANGUAGES = {
         Language.CS: "cs",
         Language.DE: "de",
@@ -76,7 +76,7 @@ class XTTSService(TTSService):
         base_url: str,
         aiohttp_session: aiohttp.ClientSession,
         language: Language = Language.EN,
-        sample_rate: int = 24000,
+        sample_rate: Optional[int] = None,
         **kwargs,
     ):
         super().__init__(sample_rate=sample_rate, **kwargs)
@@ -86,7 +86,7 @@ class XTTSService(TTSService):
             "base_url": base_url,
         }
         self.set_voice(voice_id)
-        self._studio_speakers: Dict[str, Any] | None = None
+        self._studio_speakers: Optional[Dict[str, Any]] = None
         self._aiohttp_session = aiohttp_session
 
         self._resampler = create_default_resampler()
@@ -94,11 +94,15 @@ class XTTSService(TTSService):
     def can_generate_metrics(self) -> bool:
         return True
 
-    def language_to_service_language(self, language: Language) -> str | None:
+    def language_to_service_language(self, language: Language) -> Optional[str]:
         return language_to_xtts_language(language)
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+
+        if self._studio_speakers:
+            return
+
         async with self._aiohttp_session.get(self._settings["base_url"] + "/studio_speakers") as r:
             if r.status != 200:
                 text = await r.text()
@@ -114,7 +118,7 @@ class XTTSService(TTSService):
             self._studio_speakers = await r.json()
 
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
-        logger.debug(f"Generating TTS: [{text}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         if not self._studio_speakers:
             logger.error(f"{self} no studio speakers available")
@@ -146,8 +150,10 @@ class XTTSService(TTSService):
 
             yield TTSStartedFrame()
 
+            CHUNK_SIZE = 1024
+
             buffer = bytearray()
-            async for chunk in r.content.iter_chunked(1024):
+            async for chunk in r.content.iter_chunked(CHUNK_SIZE):
                 if len(chunk) > 0:
                     await self.stop_ttfb_metrics()
                     # Append new chunk to the buffer.
@@ -164,18 +170,18 @@ class XTTSService(TTSService):
 
                         # XTTS uses 24000 so we need to resample to our desired rate.
                         resampled_audio = await self._resampler.resample(
-                            bytes(process_data), 24000, self._sample_rate
+                            bytes(process_data), 24000, self.sample_rate
                         )
                         # Create the frame with the resampled audio
-                        frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)
+                        frame = TTSAudioRawFrame(resampled_audio, self.sample_rate, 1)
                         yield frame
 
             # Process any remaining data in the buffer.
             if len(buffer) > 0:
                 resampled_audio = await self._resampler.resample(
-                    bytes(buffer), 24000, self._sample_rate
+                    bytes(buffer), 24000, self.sample_rate
                 )
-                frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)
+                frame = TTSAudioRawFrame(resampled_audio, self.sample_rate, 1)
                 yield frame
 
             yield TTSStoppedFrame()

src/pipecat/tests/utils.py

@@ -6,23 +6,30 @@
 
 import asyncio
 from dataclasses import dataclass
-from typing import Awaitable, Callable, Sequence, Tuple
+from typing import Any, Awaitable, Callable, Dict, Optional, Sequence, Tuple
 
-from pipecat.clocks.system_clock import SystemClock
 from pipecat.frames.frames import (
-    ControlFrame,
+    EndFrame,
     Frame,
     HeartbeatFrame,
     StartFrame,
+    SystemFrame,
 )
 from pipecat.observers.base_observer import BaseObserver
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.utils.asyncio import TaskManager
 
 
 @dataclass
-class EndTestFrame(ControlFrame):
-    pass
+class SleepFrame(SystemFrame):
+    """This frame is used by test framework to introduce some sleep time before
+    the next frame is pushed. This is useful to control system frames vs data or
+    control frames.
+    """
+
+    sleep: float = 0.1
 
 
 class HeartbeatsObserver(BaseObserver):
@@ -48,79 +55,106 @@ class HeartbeatsObserver(BaseObserver):
 
 
 class QueuedFrameProcessor(FrameProcessor):
-    def __init__(self, queue: asyncio.Queue, ignore_start: bool = True):
+    def __init__(
+        self,
+        *,
+        queue: asyncio.Queue,
+        queue_direction: FrameDirection,
+        ignore_start: bool = True,
+    ):
         super().__init__()
         self._queue = queue
+        self._queue_direction = queue_direction
         self._ignore_start = ignore_start
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
 
-        if self._ignore_start and isinstance(frame, StartFrame):
-            await self.push_frame(frame, direction)
-        else:
-            await self._queue.put(frame)
-            await self.push_frame(frame, direction)
+        if direction == self._queue_direction:
+            if not isinstance(frame, StartFrame) or not self._ignore_start:
+                await self._queue.put(frame)
+        await self.push_frame(frame, direction)
 
 
 async def run_test(
     processor: FrameProcessor,
+    *,
     frames_to_send: Sequence[Frame],
-    expected_down_frames: Sequence[type],
-    expected_up_frames: Sequence[type] = [],
+    expected_down_frames: Optional[Sequence[type]] = None,
+    expected_up_frames: Optional[Sequence[type]] = None,
+    ignore_start: bool = True,
+    start_metadata: Dict[str, Any] = {},
+    send_end_frame: bool = True,
 ) -> Tuple[Sequence[Frame], Sequence[Frame]]:
     received_up = asyncio.Queue()
     received_down = asyncio.Queue()
-    source = QueuedFrameProcessor(received_up)
-    sink = QueuedFrameProcessor(received_down)
+    source = QueuedFrameProcessor(
+        queue=received_up,
+        queue_direction=FrameDirection.UPSTREAM,
+        ignore_start=ignore_start,
+    )
+    sink = QueuedFrameProcessor(
+        queue=received_down,
+        queue_direction=FrameDirection.DOWNSTREAM,
+        ignore_start=ignore_start,
+    )
 
-    source.link(processor)
-    processor.link(sink)
+    pipeline = Pipeline([source, processor, sink])
 
-    task_manager = TaskManager()
-    task_manager.set_event_loop(asyncio.get_event_loop())
-    await source.queue_frame(StartFrame(clock=SystemClock(), task_manager=task_manager))
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(start_metadata=start_metadata),
+        cancel_on_idle_timeout=False,
+    )
 
-    for frame in frames_to_send:
-        await processor.process_frame(frame, FrameDirection.DOWNSTREAM)
+    async def push_frames():
+        # Just give a little head start to the runner.
+        await asyncio.sleep(0.01)
+        for frame in frames_to_send:
+            if isinstance(frame, SleepFrame):
+                await asyncio.sleep(frame.sleep)
+            else:
+                await task.queue_frame(frame)
 
-    await processor.queue_frame(EndTestFrame())
-    await processor.queue_frame(EndTestFrame(), FrameDirection.UPSTREAM)
+        if send_end_frame:
+            await task.queue_frame(EndFrame())
+
+    runner = PipelineRunner()
+    await asyncio.gather(runner.run(task), push_frames())
 
     #
     # Down frames
     #
     received_down_frames: Sequence[Frame] = []
-    running = True
-    while running:
-        frame = await received_down.get()
-        running = not isinstance(frame, EndTestFrame)
-        if running:
-            received_down_frames.append(frame)
+    if expected_down_frames is not None:
+        while not received_down.empty():
+            frame = await received_down.get()
+            if not isinstance(frame, EndFrame) or not send_end_frame:
+                received_down_frames.append(frame)
 
-    print("received DOWN frames =", received_down_frames)
+        print("received DOWN frames =", received_down_frames)
+        print("expected DOWN frames =", expected_down_frames)
 
-    assert len(received_down_frames) == len(expected_down_frames)
+        assert len(received_down_frames) == len(expected_down_frames)
 
-    for real, expected in zip(received_down_frames, expected_down_frames):
-        assert isinstance(real, expected)
+        for real, expected in zip(received_down_frames, expected_down_frames):
+            assert isinstance(real, expected)
 
     #
     # Up frames
     #
     received_up_frames: Sequence[Frame] = []
-    running = True
-    while running:
-        frame = await received_up.get()
-        running = not isinstance(frame, EndTestFrame)
-        if running:
+    if expected_up_frames is not None:
+        while not received_up.empty():
+            frame = await received_up.get()
             received_up_frames.append(frame)
 
-    print("received UP frames =", received_up_frames)
+        print("received UP frames =", received_up_frames)
+        print("expected UP frames =", expected_up_frames)
 
-    assert len(received_up_frames) == len(expected_up_frames)
+        assert len(received_up_frames) == len(expected_up_frames)
 
-    for real, expected in zip(received_up_frames, expected_up_frames):
-        assert isinstance(real, expected)
+        for real, expected in zip(received_up_frames, expected_up_frames):
+            assert isinstance(real, expected)
 
     return (received_down_frames, received_up_frames)

src/pipecat/transcriptions/language.py

@@ -54,6 +54,12 @@ class Language(StrEnum):
     AZ = "az"
     AZ_AZ = "az-AZ"
 
+    # Bashkir
+    BA = "ba"
+
+    # Belarusian
+    BE = "be"
+
     # Bulgarian
     BG = "bg"
     BG_BG = "bg-BG"
@@ -63,6 +69,12 @@ class Language(StrEnum):
     BN_BD = "bn-BD"
     BN_IN = "bn-IN"
 
+    # Tibetan
+    BO = "bo"
+
+    # Breton
+    BR = "br"
+
     # Bosnian
     BS = "bs"
     BS_BA = "bs-BA"
@@ -98,6 +110,7 @@ class Language(StrEnum):
     EN_AU = "en-AU"
     EN_CA = "en-CA"
     EN_GB = "en-GB"
+    EN_GH = "en-GH"
     EN_HK = "en-HK"
     EN_IE = "en-IE"
     EN_IN = "en-IN"
@@ -155,6 +168,9 @@ class Language(StrEnum):
     FIL = "fil"
     FIL_PH = "fil-PH"
 
+    # Faroese
+    FO = "fo"
+
     # French
     FR = "fr"
     FR_BE = "fr-BE"
@@ -174,6 +190,9 @@ class Language(StrEnum):
     GU = "gu"
     GU_IN = "gu-IN"
 
+    # Hausa
+    HA = "ha"
+
     # Hebrew
     HE = "he"
     HE_IL = "he-IL"
@@ -186,6 +205,9 @@ class Language(StrEnum):
     HR = "hr"
     HR_HR = "hr-HR"
 
+    # Haitian Creole
+    HT = "ht"
+
     # Hungarian
     HU = "hu"
     HU_HU = "hu-HU"
@@ -205,6 +227,7 @@ class Language(StrEnum):
     # Italian
     IT = "it"
     IT_IT = "it-IT"
+    IT_CH = "it-CH"
 
     # Inuktitut
     IU_CANS = "iu-Cans"
@@ -219,6 +242,7 @@ class Language(StrEnum):
     # Javanese
     JV = "jv"
     JV_ID = "jv-ID"
+    JW = "jw"  # Fal requires for Javanese
 
     # Georgian
     KA = "ka"
@@ -240,6 +264,15 @@ class Language(StrEnum):
     KO = "ko"
     KO_KR = "ko-KR"
 
+    # Latin
+    LA = "la"
+
+    # Luxembourgish
+    LB = "lb"
+
+    # Lingala
+    LN = "ln"
+
     # Lao
     LO = "lo"
     LO_LA = "lo-LA"
@@ -252,6 +285,9 @@ class Language(StrEnum):
     LV = "lv"
     LV_LV = "lv-LV"
 
+    # Malagasy
+    MG = "mg"
+
     # Macedonian
     MK = "mk"
     MK_MK = "mk-MK"
@@ -264,6 +300,9 @@ class Language(StrEnum):
     MN = "mn"
     MN_MN = "mn-MN"
 
+    # Maori
+    MI = "mi"
+
     # Marathi
     MR = "mr"
     MR_IN = "mr-IN"
@@ -281,9 +320,10 @@ class Language(StrEnum):
     MY_MM = "my-MM"
 
     # Norwegian
-    NB = "nb"
+    NB = "nb"  # Norwegian Bokmål
     NB_NO = "nb-NO"
     NO = "no"
+    NN = "nn"  # Norwegian Nynorsk
 
     # Nepali
     NE = "ne"
@@ -294,6 +334,9 @@ class Language(StrEnum):
     NL_BE = "nl-BE"
     NL_NL = "nl-NL"
 
+    # Occitan
+    OC = "oc"
+
     # Odia
     OR = "or"
     OR_IN = "or-IN"
@@ -323,6 +366,12 @@ class Language(StrEnum):
     RU = "ru"
     RU_RU = "ru-RU"
 
+    # Sanskrit
+    SA = "sa"
+
+    # Sindhi
+    SD = "sd"
+
     # Sinhala
     SI = "si"
     SI_LK = "si-LK"
@@ -335,6 +384,9 @@ class Language(StrEnum):
     SL = "sl"
     SL_SI = "sl-SI"
 
+    # Shona
+    SN = "sn"
+
     # Somali
     SO = "so"
     SO_SO = "so-SO"
@@ -376,14 +428,23 @@ class Language(StrEnum):
     TE = "te"
     TE_IN = "te-IN"
 
+    # Tajik
+    TG = "tg"
+
     # Thai
     TH = "th"
     TH_TH = "th-TH"
 
+    # Turkmen
+    TK = "tk"
+
     # Turkish
     TR = "tr"
     TR_TR = "tr-TR"
 
+    # Tatar
+    TT = "tt"
+
     # Ukrainian
     UK = "uk"
     UK_UA = "uk-UA"
@@ -405,6 +466,12 @@ class Language(StrEnum):
     WUU = "wuu"
     WUU_CN = "wuu-CN"
 
+    # Yiddish
+    YI = "yi"
+
+    # Yoruba
+    YO = "yo"
+
     # Yue Chinese
     YUE = "yue"
     YUE_CN = "yue-CN"

src/pipecat/transports/base_input.py

@@ -6,6 +6,7 @@
 
 import asyncio
 from concurrent.futures import ThreadPoolExecutor
+from typing import Optional
 
 from loguru import logger
 
@@ -13,6 +14,8 @@ from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADState
 from pipecat.frames.frames import (
     BotInterruptionFrame,
     CancelFrame,
+    EmulateUserStartedSpeakingFrame,
+    EmulateUserStoppedSpeakingFrame,
     EndFrame,
     FilterUpdateSettingsFrame,
     Frame,
@@ -35,6 +38,9 @@ class BaseInputTransport(FrameProcessor):
 
         self._params = params
 
+        # Input sample rate. It will be initialized on StartFrame.
+        self._sample_rate = 0
+
         # We read audio from a single queue one at a time and we then run VAD in
         # a thread. Therefore, only one thread should be necessary.
         self._executor = ThreadPoolExecutor(max_workers=1)
@@ -43,12 +49,32 @@ class BaseInputTransport(FrameProcessor):
         # if passthrough is enabled.
         self._audio_task = None
 
+    def enable_audio_in_stream_on_start(self, enabled: bool) -> None:
+        logger.debug(f"Enabling audio on start. {enabled}")
+        self._params.audio_in_stream_on_start = enabled
+
+    def start_audio_in_streaming(self):
+        pass
+
+    @property
+    def sample_rate(self) -> int:
+        return self._sample_rate
+
+    @property
+    def vad_analyzer(self) -> Optional[VADAnalyzer]:
+        return self._params.vad_analyzer
+
     async def start(self, frame: StartFrame):
+        self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
+
+        # Configure VAD analyzer.
+        if self._params.vad_enabled and self._params.vad_analyzer:
+            self._params.vad_analyzer.set_sample_rate(self._sample_rate)
         # Start audio filter.
         if self._params.audio_in_filter:
-            await self._params.audio_in_filter.start(self._params.audio_in_sample_rate)
+            await self._params.audio_in_filter.start(self._sample_rate)
         # Create audio input queue and task if needed.
-        if self._params.audio_in_enabled or self._params.vad_enabled:
+        if not self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled):
             self._audio_in_queue = asyncio.Queue()
             self._audio_task = self.create_task(self._audio_task_handler())
 
@@ -67,9 +93,6 @@ class BaseInputTransport(FrameProcessor):
             await self.cancel_task(self._audio_task)
             self._audio_task = None
 
-    def vad_analyzer(self) -> VADAnalyzer | None:
-        return self._params.vad_analyzer
-
     async def push_audio_frame(self, frame: InputAudioRawFrame):
         if self._params.audio_in_enabled or self._params.vad_enabled:
             await self._audio_in_queue.put(frame)
@@ -91,9 +114,13 @@ class BaseInputTransport(FrameProcessor):
             await self.cancel(frame)
             await self.push_frame(frame, direction)
         elif isinstance(frame, BotInterruptionFrame):
-            logger.debug("Bot interruption")
-            await self._start_interruption()
-            await self.push_frame(StartInterruptionFrame())
+            await self._handle_bot_interruption(frame)
+        elif isinstance(frame, EmulateUserStartedSpeakingFrame):
+            logger.debug("Emulating user started speaking")
+            await self._handle_user_interruption(UserStartedSpeakingFrame())
+        elif isinstance(frame, EmulateUserStoppedSpeakingFrame):
+            logger.debug("Emulating user stopped speaking")
+            await self._handle_user_interruption(UserStoppedSpeakingFrame())
         # All other system frames
         elif isinstance(frame, SystemFrame):
             await self.push_frame(frame, direction)
@@ -104,9 +131,8 @@ class BaseInputTransport(FrameProcessor):
             await self.push_frame(frame, direction)
             await self.stop(frame)
         elif isinstance(frame, VADParamsUpdateFrame):
-            vad_analyzer = self.vad_analyzer()
-            if vad_analyzer:
-                vad_analyzer.set_params(frame.params)
+            if self.vad_analyzer:
+                self.vad_analyzer.set_params(frame.params)
         elif isinstance(frame, FilterUpdateSettingsFrame) and self._params.audio_in_filter:
             await self._params.audio_in_filter.process_frame(frame)
         # Other frames
@@ -117,36 +143,40 @@ class BaseInputTransport(FrameProcessor):
     # Handle interruptions
     #
 
-    async def _handle_interruptions(self, frame: Frame):
+    async def _handle_bot_interruption(self, frame: BotInterruptionFrame):
+        logger.debug("Bot interruption")
         if self.interruptions_allowed:
+            await self._start_interruption()
+            await self.push_frame(StartInterruptionFrame())
+
+    async def _handle_user_interruption(self, frame: Frame):
+        if isinstance(frame, UserStartedSpeakingFrame):
+            logger.debug("User started speaking")
+            await self.push_frame(frame)
             # Make sure we notify about interruptions quickly out-of-band.
-            if isinstance(frame, UserStartedSpeakingFrame):
-                logger.debug("User started speaking")
+            if self.interruptions_allowed:
                 await self._start_interruption()
                 # Push an out-of-band frame (i.e. not using the ordered push
                 # frame task) to stop everything, specially at the output
                 # transport.
                 await self.push_frame(StartInterruptionFrame())
-            elif isinstance(frame, UserStoppedSpeakingFrame):
-                logger.debug("User stopped speaking")
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            logger.debug("User stopped speaking")
+            await self.push_frame(frame)
+            if self.interruptions_allowed:
                 await self._stop_interruption()
                 await self.push_frame(StopInterruptionFrame())
 
-        await self.push_frame(frame)
-
     #
     # Audio input
     #
 
     async def _vad_analyze(self, audio_frame: InputAudioRawFrame) -> VADState:
         state = VADState.QUIET
-        vad_analyzer = self.vad_analyzer()
-        if vad_analyzer:
-            logger.trace(f"{self}: analyzing VAD on {audio_frame}")
+        if self.vad_analyzer:
             state = await self.get_event_loop().run_in_executor(
-                self._executor, vad_analyzer.analyze_audio, audio_frame.audio
+                self._executor, self.vad_analyzer.analyze_audio, audio_frame.audio
             )
-            logger.trace(f"{self}: done analyzing VAD on {audio_frame}")
         return state
 
     async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState):
@@ -163,7 +193,7 @@ class BaseInputTransport(FrameProcessor):
                 frame = UserStoppedSpeakingFrame()
 
             if frame:
-                await self._handle_interruptions(frame)
+                await self._handle_user_interruption(frame)
 
             vad_state = new_vad_state
         return vad_state

src/pipecat/transports/base_output.py

@@ -13,7 +13,7 @@ from typing import AsyncGenerator, List
 from loguru import logger
 from PIL import Image
 
-from pipecat.audio.vad.vad_analyzer import VAD_STOP_SECS
+from pipecat.audio.utils import create_default_resampler
 from pipecat.frames.frames import (
     BotSpeakingFrame,
     BotStartedSpeakingFrame,
@@ -37,6 +37,8 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.transports.base_transport import TransportParams
 from pipecat.utils.time import nanoseconds_to_seconds
 
+BOT_VAD_STOP_SECS = 0.3
+
 
 class BaseOutputTransport(FrameProcessor):
     def __init__(self, params: TransportParams, **kwargs):
@@ -57,12 +59,12 @@ class BaseOutputTransport(FrameProcessor):
         # framerate.
         self._camera_images = None
 
-        # We will write 20ms audio at a time. If we receive long audio frames we
-        # will chunk them. This will help with interruption handling.
-        audio_bytes_10ms = (
-            int(self._params.audio_out_sample_rate / 100) * self._params.audio_out_channels * 2
-        )
-        self._audio_chunk_size = audio_bytes_10ms * 2
+        # Output sample rate. It will be initialized on StartFrame.
+        self._sample_rate = 0
+        self._resampler = create_default_resampler()
+
+        # Chunk size that will be written. It will be computed on StartFrame
+        self._audio_chunk_size = 0
         self._audio_buffer = bytearray()
 
         self._stopped_event = asyncio.Event()
@@ -70,10 +72,21 @@ class BaseOutputTransport(FrameProcessor):
         # Indicates if the bot is currently speaking.
         self._bot_speaking = False
 
+    @property
+    def sample_rate(self) -> int:
+        return self._sample_rate
+
     async def start(self, frame: StartFrame):
+        self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
+
+        # We will write 20ms audio at a time. If we receive long audio frames we
+        # will chunk them. This will help with interruption handling.
+        audio_bytes_10ms = int(self._sample_rate / 100) * self._params.audio_out_channels * 2
+        self._audio_chunk_size = audio_bytes_10ms * 2
+
         # Start audio mixer.
         if self._params.audio_out_mixer:
-            await self._params.audio_out_mixer.start(self._params.audio_out_sample_rate)
+            await self._params.audio_out_mixer.start(self._sample_rate)
         self._create_camera_task()
         self._create_sink_tasks()
 
@@ -157,6 +170,8 @@ class BaseOutputTransport(FrameProcessor):
         # TODO(aleix): Images and audio should support presentation timestamps.
         elif frame.pts:
             await self._sink_clock_queue.put((frame.pts, frame.id, frame))
+        elif direction == FrameDirection.UPSTREAM:
+            await self.push_frame(frame, direction)
         else:
             await self._sink_queue.put(frame)
 
@@ -178,12 +193,18 @@ class BaseOutputTransport(FrameProcessor):
         if not self._params.audio_out_enabled:
             return
 
+        # We might need to resample if incoming audio doesn't match the
+        # transport sample rate.
+        resampled = await self._resampler.resample(
+            frame.audio, frame.sample_rate, self._sample_rate
+        )
+
         cls = type(frame)
-        self._audio_buffer.extend(frame.audio)
+        self._audio_buffer.extend(resampled)
         while len(self._audio_buffer) >= self._audio_chunk_size:
             chunk = cls(
                 bytes(self._audio_buffer[: self._audio_chunk_size]),
-                sample_rate=frame.sample_rate,
+                sample_rate=self._sample_rate,
                 num_channels=frame.num_channels,
             )
             await self._sink_queue.put(chunk)
@@ -211,16 +232,21 @@ class BaseOutputTransport(FrameProcessor):
             await self.push_frame(BotStoppedSpeakingFrame())
             await self.push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM)
             self._bot_speaking = False
+            # Clean audio buffer (there could be tiny left overs if not multiple
+            # to our output chunk size).
+            self._audio_buffer = bytearray()
 
     #
     # Sink tasks
     #
 
     def _create_sink_tasks(self):
-        self._sink_queue = asyncio.Queue()
-        self._sink_clock_queue = asyncio.PriorityQueue()
-        self._sink_task = self.create_task(self._sink_task_handler())
-        self._sink_clock_task = self.create_task(self._sink_clock_task_handler())
+        if not self._sink_task:
+            self._sink_queue = asyncio.Queue()
+            self._sink_task = self.create_task(self._sink_task_handler())
+        if not self._sink_clock_task:
+            self._sink_clock_queue = asyncio.PriorityQueue()
+            self._sink_clock_task = self.create_task(self._sink_clock_task_handler())
 
     async def _cancel_sink_tasks(self):
         # Stop sink tasks.
@@ -298,20 +324,15 @@ class BaseOutputTransport(FrameProcessor):
                     # Generate an audio frame with only the mixer's part.
                     frame = OutputAudioRawFrame(
                         audio=await self._params.audio_out_mixer.mix(silence),
-                        sample_rate=self._params.audio_out_sample_rate,
+                        sample_rate=self._sample_rate,
                         num_channels=self._params.audio_out_channels,
                     )
                     yield frame
 
-        vad_stop_secs = (
-            self._params.vad_analyzer.params.stop_secs
-            if self._params.vad_analyzer
-            else VAD_STOP_SECS
-        )
         if self._params.audio_out_mixer:
-            return with_mixer(vad_stop_secs)
+            return with_mixer(BOT_VAD_STOP_SECS)
         else:
-            return without_mixer(vad_stop_secs)
+            return without_mixer(BOT_VAD_STOP_SECS)
 
     async def _sink_task_handler(self):
         async for frame in self._next_frame():
@@ -342,7 +363,7 @@ class BaseOutputTransport(FrameProcessor):
 
     def _create_camera_task(self):
         # Create camera output queue and task if needed.
-        if self._params.camera_out_enabled:
+        if not self._camera_out_task and self._params.camera_out_enabled:
             self._camera_out_queue = asyncio.Queue()
             self._camera_out_task = self.create_task(self._camera_out_task_handler())
 

src/pipecat/transports/base_transport.py

@@ -4,19 +4,16 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-import asyncio
-import inspect
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from typing import Optional
 
-from loguru import logger
 from pydantic import BaseModel, ConfigDict
 
 from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
 from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer
 from pipecat.audio.vad.vad_analyzer import VADAnalyzer
 from pipecat.processors.frame_processor import FrameProcessor
-from pipecat.utils.utils import obj_count, obj_id
+from pipecat.utils.base_object import BaseObject
 
 
 class TransportParams(BaseModel):
@@ -30,21 +27,21 @@ class TransportParams(BaseModel):
     camera_out_framerate: int = 30
     camera_out_color_format: str = "RGB"
     audio_out_enabled: bool = False
-    audio_out_is_live: bool = False
-    audio_out_sample_rate: int = 24000
+    audio_out_sample_rate: Optional[int] = None
     audio_out_channels: int = 1
     audio_out_bitrate: int = 96000
     audio_out_mixer: Optional[BaseAudioMixer] = None
     audio_in_enabled: bool = False
-    audio_in_sample_rate: int = 16000
+    audio_in_sample_rate: Optional[int] = None
     audio_in_channels: int = 1
     audio_in_filter: Optional[BaseAudioFilter] = None
+    audio_in_stream_on_start: bool = True
     vad_enabled: bool = False
     vad_audio_passthrough: bool = False
-    vad_analyzer: VADAnalyzer | None = None
+    vad_analyzer: Optional[VADAnalyzer] = None
 
 
-class BaseTransport(ABC):
+class BaseTransport(BaseObject):
     def __init__(
         self,
         *,
@@ -52,54 +49,14 @@ class BaseTransport(ABC):
         input_name: Optional[str] = None,
         output_name: Optional[str] = None,
     ):
-        self._id: int = obj_id()
-        self._name = name or f"{self.__class__.__name__}#{obj_count(self)}"
+        super().__init__(name=name)
         self._input_name = input_name
         self._output_name = output_name
-        self._event_handlers: dict = {}
-
-    @property
-    def id(self) -> int:
-        return self._id
-
-    @property
-    def name(self) -> str:
-        return self._name
 
     @abstractmethod
     def input(self) -> FrameProcessor:
-        raise NotImplementedError
+        pass
 
     @abstractmethod
     def output(self) -> FrameProcessor:
-        raise NotImplementedError
-
-    def event_handler(self, event_name: str):
-        def decorator(handler):
-            self.add_event_handler(event_name, handler)
-            return handler
-
-        return decorator
-
-    def add_event_handler(self, event_name: str, handler):
-        if event_name not in self._event_handlers:
-            raise Exception(f"Event handler {event_name} not registered")
-        self._event_handlers[event_name].append(handler)
-
-    def _register_event_handler(self, event_name: str):
-        if event_name in self._event_handlers:
-            raise Exception(f"Event handler {event_name} already registered")
-        self._event_handlers[event_name] = []
-
-    async def _call_event_handler(self, event_name: str, *args, **kwargs):
-        try:
-            for handler in self._event_handlers[event_name]:
-                if inspect.iscoroutinefunction(handler):
-                    await handler(self, *args, **kwargs)
-                else:
-                    handler(self, *args, **kwargs)
-        except Exception as e:
-            logger.exception(f"Exception in event handler {event_name}: {e}")
-
-    def __str__(self):
-        return self.name
+        pass

src/pipecat/transports/local/audio.py

@@ -6,6 +6,7 @@
 
 import asyncio
 from concurrent.futures import ThreadPoolExecutor
+from typing import Optional
 
 from loguru import logger
 
@@ -25,38 +26,52 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+class LocalAudioTransportParams(TransportParams):
+    input_device_index: Optional[int] = None
+    output_device_index: Optional[int] = None
+
+
 class LocalAudioInputTransport(BaseInputTransport):
-    def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
+    _params: LocalAudioTransportParams
+
+    def __init__(self, py_audio: pyaudio.PyAudio, params: LocalAudioTransportParams):
         super().__init__(params)
+        self._py_audio = py_audio
 
-        sample_rate = self._params.audio_in_sample_rate
-        num_frames = int(sample_rate / 100) * 2  # 20ms of audio
-
-        self._in_stream = py_audio.open(
-            format=py_audio.get_format_from_width(2),
-            channels=params.audio_in_channels,
-            rate=params.audio_in_sample_rate,
-            frames_per_buffer=num_frames,
-            stream_callback=self._audio_in_callback,
-            input=True,
-        )
+        self._in_stream = None
+        self._sample_rate = 0
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+
+        if self._in_stream:
+            return
+
+        self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
+        num_frames = int(self._sample_rate / 100) * 2  # 20ms of audio
+
+        self._in_stream = self._py_audio.open(
+            format=self._py_audio.get_format_from_width(2),
+            channels=self._params.audio_in_channels,
+            rate=self._sample_rate,
+            frames_per_buffer=num_frames,
+            stream_callback=self._audio_in_callback,
+            input=True,
+            input_device_index=self._params.input_device_index,
+        )
         self._in_stream.start_stream()
 
     async def cleanup(self):
         await super().cleanup()
-        self._in_stream.stop_stream()
-        # This is not very pretty (taken from PyAudio docs).
-        while self._in_stream.is_active():
-            await asyncio.sleep(0.1)
-        self._in_stream.close()
+        if self._in_stream:
+            self._in_stream.stop_stream()
+            self._in_stream.close()
+            self._in_stream = None
 
     def _audio_in_callback(self, in_data, frame_count, time_info, status):
         frame = InputAudioRawFrame(
             audio=in_data,
-            sample_rate=self._params.audio_in_sample_rate,
+            sample_rate=self._sample_rate,
             num_channels=self._params.audio_in_channels,
         )
 
@@ -66,44 +81,58 @@ class LocalAudioInputTransport(BaseInputTransport):
 
 
 class LocalAudioOutputTransport(BaseOutputTransport):
-    def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
+    _params: LocalAudioTransportParams
+
+    def __init__(self, py_audio: pyaudio.PyAudio, params: LocalAudioTransportParams):
         super().__init__(params)
+        self._py_audio = py_audio
+
+        self._out_stream = None
+        self._sample_rate = 0
 
         # We only write audio frames from a single task, so only one thread
         # should be necessary.
         self._executor = ThreadPoolExecutor(max_workers=1)
 
-        self._out_stream = py_audio.open(
-            format=py_audio.get_format_from_width(2),
-            channels=params.audio_out_channels,
-            rate=params.audio_out_sample_rate,
-            output=True,
-        )
-
     async def start(self, frame: StartFrame):
         await super().start(frame)
+
+        if self._out_stream:
+            return
+
+        self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
+
+        self._out_stream = self._py_audio.open(
+            format=self._py_audio.get_format_from_width(2),
+            channels=self._params.audio_out_channels,
+            rate=self._sample_rate,
+            output=True,
+            output_device_index=self._params.output_device_index,
+        )
         self._out_stream.start_stream()
 
     async def cleanup(self):
         await super().cleanup()
-        self._out_stream.stop_stream()
-        # This is not very pretty (taken from PyAudio docs).
-        while self._out_stream.is_active():
-            await asyncio.sleep(0.1)
-        self._out_stream.close()
+        if self._out_stream:
+            self._out_stream.stop_stream()
+            self._out_stream.close()
+            self._out_stream = None
 
     async def write_raw_audio_frames(self, frames: bytes):
-        await self.get_event_loop().run_in_executor(self._executor, self._out_stream.write, frames)
+        if self._out_stream:
+            await self.get_event_loop().run_in_executor(
+                self._executor, self._out_stream.write, frames
+            )
 
 
 class LocalAudioTransport(BaseTransport):
-    def __init__(self, params: TransportParams):
+    def __init__(self, params: LocalAudioTransportParams):
         super().__init__()
         self._params = params
         self._pyaudio = pyaudio.PyAudio()
 
-        self._input: LocalAudioInputTransport | None = None
-        self._output: LocalAudioOutputTransport | None = None
+        self._input: Optional[LocalAudioInputTransport] = None
+        self._output: Optional[LocalAudioOutputTransport] = None
 
     #
     # BaseTransport

src/pipecat/transports/local/tk.py

@@ -7,6 +7,7 @@
 import asyncio
 import tkinter as tk
 from concurrent.futures import ThreadPoolExecutor
+from typing import Optional
 
 import numpy as np
 from loguru import logger
@@ -33,38 +34,51 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+class TkTransportParams(TransportParams):
+    audio_input_device_index: Optional[int] = None
+    audio_output_device_index: Optional[int] = None
+
+
 class TkInputTransport(BaseInputTransport):
-    def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
+    _params: TkTransportParams
+
+    def __init__(self, py_audio: pyaudio.PyAudio, params: TkTransportParams):
         super().__init__(params)
-
-        sample_rate = self._params.audio_in_sample_rate
-        num_frames = int(sample_rate / 100) * 2  # 20ms of audio
-
-        self._in_stream = py_audio.open(
-            format=py_audio.get_format_from_width(2),
-            channels=params.audio_in_channels,
-            rate=params.audio_in_sample_rate,
-            frames_per_buffer=num_frames,
-            stream_callback=self._audio_in_callback,
-            input=True,
-        )
+        self._py_audio = py_audio
+        self._in_stream = None
+        self._sample_rate = 0
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+
+        if self._in_stream:
+            return
+
+        self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
+        num_frames = int(self._sample_rate / 100) * 2  # 20ms of audio
+
+        self._in_stream = self._py_audio.open(
+            format=self._py_audio.get_format_from_width(2),
+            channels=self._params.audio_in_channels,
+            rate=self._sample_rate,
+            frames_per_buffer=num_frames,
+            stream_callback=self._audio_in_callback,
+            input=True,
+            input_device_index=self._params.audio_input_device_index,
+        )
         self._in_stream.start_stream()
 
     async def cleanup(self):
         await super().cleanup()
-        self._in_stream.stop_stream()
-        # This is not very pretty (taken from PyAudio docs).
-        while self._in_stream.is_active():
-            await asyncio.sleep(0.1)
-        self._in_stream.close()
+        if self._in_stream:
+            self._in_stream.stop_stream()
+            self._in_stream.close()
+            self._in_stream = None
 
     def _audio_in_callback(self, in_data, frame_count, time_info, status):
         frame = InputAudioRawFrame(
             audio=in_data,
-            sample_rate=self._params.audio_in_sample_rate,
+            sample_rate=self._sample_rate,
             num_channels=self._params.audio_in_channels,
         )
 
@@ -74,20 +88,18 @@ class TkInputTransport(BaseInputTransport):
 
 
 class TkOutputTransport(BaseOutputTransport):
-    def __init__(self, tk_root: tk.Tk, py_audio: pyaudio.PyAudio, params: TransportParams):
+    _params: TkTransportParams
+
+    def __init__(self, tk_root: tk.Tk, py_audio: pyaudio.PyAudio, params: TkTransportParams):
         super().__init__(params)
+        self._py_audio = py_audio
+        self._out_stream = None
+        self._sample_rate = 0
 
         # We only write audio frames from a single task, so only one thread
         # should be necessary.
         self._executor = ThreadPoolExecutor(max_workers=1)
 
-        self._out_stream = py_audio.open(
-            format=py_audio.get_format_from_width(2),
-            channels=params.audio_out_channels,
-            rate=params.audio_out_sample_rate,
-            output=True,
-        )
-
         # Start with a neutral gray background.
         array = np.ones((1024, 1024, 3)) * 128
         data = f"P5 {1024} {1024} 255 ".encode() + array.astype(np.uint8).tobytes()
@@ -97,18 +109,33 @@ class TkOutputTransport(BaseOutputTransport):
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+
+        if self._out_stream:
+            return
+
+        self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
+
+        self._out_stream = self._py_audio.open(
+            format=self._py_audio.get_format_from_width(2),
+            channels=self._params.audio_out_channels,
+            rate=self._sample_rate,
+            output=True,
+            output_device_index=self._params.audio_output_device_index,
+        )
         self._out_stream.start_stream()
 
     async def cleanup(self):
         await super().cleanup()
-        self._out_stream.stop_stream()
-        # This is not very pretty (taken from PyAudio docs).
-        while self._out_stream.is_active():
-            await asyncio.sleep(0.1)
-        self._out_stream.close()
+        if self._out_stream:
+            self._out_stream.stop_stream()
+            self._out_stream.close()
+            self._out_stream = None
 
     async def write_raw_audio_frames(self, frames: bytes):
-        await self.get_event_loop().run_in_executor(self._executor, self._out_stream.write, frames)
+        if self._out_stream:
+            await self.get_event_loop().run_in_executor(
+                self._executor, self._out_stream.write, frames
+            )
 
     async def write_frame_to_camera(self, frame: OutputImageRawFrame):
         self.get_event_loop().call_soon(self._write_frame_to_tk, frame)
@@ -126,14 +153,14 @@ class TkOutputTransport(BaseOutputTransport):
 
 
 class TkLocalTransport(BaseTransport):
-    def __init__(self, tk_root: tk.Tk, params: TransportParams):
+    def __init__(self, tk_root: tk.Tk, params: TkTransportParams):
         super().__init__()
         self._tk_root = tk_root
         self._params = params
         self._pyaudio = pyaudio.PyAudio()
 
-        self._input: TkInputTransport | None = None
-        self._output: TkOutputTransport | None = None
+        self._input: Optional[TkInputTransport] = None
+        self._output: Optional[TkOutputTransport] = None
 
     #
     # BaseTransport

src/pipecat/transports/network/fastapi_websocket.py

@@ -10,7 +10,7 @@ import io
 import time
 import typing
 import wave
-from typing import Awaitable, Callable
+from typing import Awaitable, Callable, Optional
 
 from loguru import logger
 from pydantic import BaseModel
@@ -23,6 +23,8 @@ from pipecat.frames.frames import (
     OutputAudioRawFrame,
     StartFrame,
     StartInterruptionFrame,
+    TransportMessageFrame,
+    TransportMessageUrgentFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
@@ -44,7 +46,7 @@ except ModuleNotFoundError as e:
 class FastAPIWebsocketParams(TransportParams):
     add_wav_header: bool = False
     serializer: FrameSerializer
-    session_timeout: int | None = None
+    session_timeout: Optional[int] = None
 
 
 class FastAPIWebsocketCallbacks(BaseModel):
@@ -53,44 +55,99 @@ class FastAPIWebsocketCallbacks(BaseModel):
     on_session_timeout: Callable[[WebSocket], Awaitable[None]]
 
 
+class FastAPIWebsocketClient:
+    def __init__(self, websocket: WebSocket, is_binary: bool, callbacks: FastAPIWebsocketCallbacks):
+        self._websocket = websocket
+        self._closing = False
+        self._is_binary = is_binary
+        self._callbacks = callbacks
+
+    def receive(self) -> typing.AsyncIterator[bytes | str]:
+        return self._websocket.iter_bytes() if self._is_binary else self._websocket.iter_text()
+
+    async def send(self, data: str | bytes):
+        if self._can_send():
+            if self._is_binary:
+                await self._websocket.send_bytes(data)
+            else:
+                await self._websocket.send_text(data)
+
+    async def disconnect(self):
+        if self.is_connected and not self.is_closing:
+            self._closing = True
+            await self._websocket.close()
+            await self.trigger_client_disconnected()
+
+    async def trigger_client_disconnected(self):
+        await self._callbacks.on_client_disconnected(self._websocket)
+
+    async def trigger_client_connected(self):
+        await self._callbacks.on_client_connected(self._websocket)
+
+    async def trigger_client_timout(self):
+        await self._callbacks.on_session_timeout(self._websocket)
+
+    def _can_send(self):
+        return self.is_connected and not self.is_closing
+
+    @property
+    def is_connected(self) -> bool:
+        return self._websocket.client_state == WebSocketState.CONNECTED
+
+    @property
+    def is_closing(self) -> bool:
+        return self._closing
+
+
 class FastAPIWebsocketInputTransport(BaseInputTransport):
     def __init__(
         self,
-        websocket: WebSocket,
+        transport: BaseTransport,
+        client: FastAPIWebsocketClient,
         params: FastAPIWebsocketParams,
-        callbacks: FastAPIWebsocketCallbacks,
         **kwargs,
     ):
         super().__init__(params, **kwargs)
-
-        self._websocket = websocket
+        self._transport = transport
+        self._client = client
         self._params = params
-        self._callbacks = callbacks
+        self._receive_task = None
+        self._monitor_websocket_task = None
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
-        if self._params.session_timeout:
+        await self._params.serializer.setup(frame)
+        if not self._monitor_websocket_task and self._params.session_timeout:
             self._monitor_websocket_task = self.create_task(self._monitor_websocket())
-        await self._callbacks.on_client_connected(self._websocket)
-        self._receive_task = self.create_task(self._receive_messages())
+        await self._client.trigger_client_connected()
+        if not self._receive_task:
+            self._receive_task = self.create_task(self._receive_messages())
+
+    async def _stop_tasks(self):
+        if self._monitor_websocket_task:
+            await self.cancel_task(self._monitor_websocket_task)
+            self._monitor_websocket_task = None
+        if self._receive_task:
+            await self.cancel_task(self._receive_task)
+            self._receive_task = None
 
     async def stop(self, frame: EndFrame):
         await super().stop(frame)
-        await self.cancel_task(self._receive_task)
+        await self._stop_tasks()
+        await self._client.disconnect()
 
     async def cancel(self, frame: CancelFrame):
         await super().cancel(frame)
-        await self.cancel_task(self._receive_task)
+        await self._stop_tasks()
+        await self._client.disconnect()
 
-    def _iter_data(self) -> typing.AsyncIterator[bytes | str]:
-        if self._params.serializer.type == FrameSerializerType.BINARY:
-            return self._websocket.iter_bytes()
-        else:
-            return self._websocket.iter_text()
+    async def cleanup(self):
+        await super().cleanup()
+        await self._transport.cleanup()
 
     async def _receive_messages(self):
         try:
-            async for message in self._iter_data():
+            async for message in self._client.receive():
                 frame = await self._params.serializer.deserialize(message)
 
                 if not frame:
@@ -101,26 +158,55 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
                 else:
                     await self.push_frame(frame)
         except Exception as e:
-            logger.error(f"{self} exception receiving data (class: {e.__class__.__name__})")
+            logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
 
-        await self._callbacks.on_client_disconnected(self._websocket)
+        await self._client.trigger_client_disconnected()
 
     async def _monitor_websocket(self):
         """Wait for self._params.session_timeout seconds, if the websocket is still open, trigger timeout event."""
         await asyncio.sleep(self._params.session_timeout)
-        await self._callbacks.on_session_timeout(self._websocket)
+        await self._client.trigger_client_timout()
 
 
 class FastAPIWebsocketOutputTransport(BaseOutputTransport):
-    def __init__(self, websocket: WebSocket, params: FastAPIWebsocketParams, **kwargs):
+    def __init__(
+        self,
+        transport: BaseTransport,
+        client: FastAPIWebsocketClient,
+        params: FastAPIWebsocketParams,
+        **kwargs,
+    ):
         super().__init__(params, **kwargs)
 
-        self._websocket = websocket
+        self._transport = transport
+        self._client = client
         self._params = params
 
-        self._send_interval = (self._audio_chunk_size / self._params.audio_out_sample_rate) / 2
+        # write_raw_audio_frames() is called quickly, as soon as we get audio
+        # (e.g. from the TTS), and since this is just a network connection we
+        # would be sending it to quickly. Instead, we want to block to emulate
+        # an audio device, this is what the send interval is. It will be
+        # computed on StartFrame.
+        self._send_interval = 0
         self._next_send_time = 0
 
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+        await self._params.serializer.setup(frame)
+        self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
+
+    async def stop(self, frame: EndFrame):
+        await super().stop(frame)
+        await self._client.disconnect()
+
+    async def cancel(self, frame: CancelFrame):
+        await super().cancel(frame)
+        await self._client.disconnect()
+
+    async def cleanup(self):
+        await super().cleanup()
+        await self._transport.cleanup()
+
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
 
@@ -128,15 +214,21 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
             await self._write_frame(frame)
             self._next_send_time = 0
 
+    async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
+        await self._write_frame(frame)
+
     async def write_raw_audio_frames(self, frames: bytes):
-        if self._websocket.client_state != WebSocketState.CONNECTED:
+        if self._client.is_closing:
+            return
+
+        if not self._client.is_connected:
             # Simulate audio playback with a sleep.
             await self._write_audio_sleep()
             return
 
         frame = OutputAudioRawFrame(
             audio=frames,
-            sample_rate=self._params.audio_out_sample_rate,
+            sample_rate=self.sample_rate,
             num_channels=self._params.audio_out_channels,
         )
 
@@ -156,24 +248,16 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
 
         await self._write_frame(frame)
 
-        self._websocket_audio_buffer = bytes()
-
         # Simulate audio playback with a sleep.
         await self._write_audio_sleep()
 
     async def _write_frame(self, frame: Frame):
         try:
             payload = await self._params.serializer.serialize(frame)
-            if payload and self._websocket.client_state == WebSocketState.CONNECTED:
-                await self._send_data(payload)
+            if payload:
+                await self._client.send(payload)
         except Exception as e:
-            logger.error(f"{self} exception sending data (class: {e.__class__.__name__})")
-
-    def _send_data(self, data: str | bytes):
-        if self._params.serializer.type == FrameSerializerType.BINARY:
-            return self._websocket.send_bytes(data)
-        else:
-            return self._websocket.send_text(data)
+            logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
 
     async def _write_audio_sleep(self):
         # Simulate a clock.
@@ -191,10 +275,11 @@ class FastAPIWebsocketTransport(BaseTransport):
         self,
         websocket: WebSocket,
         params: FastAPIWebsocketParams,
-        input_name: str | None = None,
-        output_name: str | None = None,
+        input_name: Optional[str] = None,
+        output_name: Optional[str] = None,
     ):
         super().__init__(input_name=input_name, output_name=output_name)
+
         self._params = params
 
         self._callbacks = FastAPIWebsocketCallbacks(
@@ -203,11 +288,14 @@ class FastAPIWebsocketTransport(BaseTransport):
             on_session_timeout=self._on_session_timeout,
         )
 
+        is_binary = self._params.serializer.type == FrameSerializerType.BINARY
+        self._client = FastAPIWebsocketClient(websocket, is_binary, self._callbacks)
+
         self._input = FastAPIWebsocketInputTransport(
-            websocket, self._params, self._callbacks, name=self._input_name
+            self, self._client, self._params, name=self._input_name
         )
         self._output = FastAPIWebsocketOutputTransport(
-            websocket, self._params, name=self._output_name
+            self, self._client, self._params, name=self._output_name
         )
 
         # Register supported handlers. The user will only be able to register

src/pipecat/transports/network/websocket_client.py

@@ -30,7 +30,7 @@ from pipecat.serializers.protobuf import ProtobufFrameSerializer
 from pipecat.transports.base_input import BaseInputTransport
 from pipecat.transports.base_output import BaseOutputTransport
 from pipecat.transports.base_transport import BaseTransport, TransportParams
-from pipecat.utils.asyncio import TaskManager
+from pipecat.utils.asyncio import BaseTaskManager
 
 
 class WebsocketClientParams(TransportParams):
@@ -57,12 +57,12 @@ class WebsocketClientSession:
         self._callbacks = callbacks
         self._transport_name = transport_name
 
-        self._task_manager: Optional[TaskManager] = None
+        self._task_manager: Optional[BaseTaskManager] = None
 
-        self._websocket: websockets.WebSocketClientProtocol | None = None
+        self._websocket: Optional[websockets.WebSocketClientProtocol] = None
 
     @property
-    def task_manager(self) -> TaskManager:
+    def task_manager(self) -> BaseTaskManager:
         if not self._task_manager:
             raise Exception(
                 f"{self._transport_name}::WebsocketClientSession: TaskManager not initialized (pipeline not started?)"
@@ -101,7 +101,7 @@ class WebsocketClientSession:
             if self._websocket:
                 await self._websocket.send(message)
         except Exception as e:
-            logger.error(f"{self} exception sending data (class: {e.__class__.__name__})")
+            logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
 
     async def _client_task_handler(self):
         try:
@@ -109,7 +109,7 @@ class WebsocketClientSession:
             async for message in self._websocket:
                 await self._callbacks.on_message(self._websocket, message)
         except Exception as e:
-            logger.error(f"{self} exception receiving data (class: {e.__class__.__name__})")
+            logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
 
         await self._callbacks.on_disconnected(self._websocket)
 
@@ -118,14 +118,21 @@ class WebsocketClientSession:
 
 
 class WebsocketClientInputTransport(BaseInputTransport):
-    def __init__(self, session: WebsocketClientSession, params: WebsocketClientParams):
+    def __init__(
+        self,
+        transport: BaseTransport,
+        session: WebsocketClientSession,
+        params: WebsocketClientParams,
+    ):
         super().__init__(params)
 
+        self._transport = transport
         self._session = session
         self._params = params
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+        await self._params.serializer.setup(frame)
         await self._session.setup(frame)
         await self._session.connect()
 
@@ -137,6 +144,10 @@ class WebsocketClientInputTransport(BaseInputTransport):
         await super().cancel(frame)
         await self._session.disconnect()
 
+    async def cleanup(self):
+        await super().cleanup()
+        await self._transport.cleanup()
+
     async def on_message(self, websocket, message):
         frame = await self._params.serializer.deserialize(message)
         if not frame:
@@ -148,17 +159,30 @@ class WebsocketClientInputTransport(BaseInputTransport):
 
 
 class WebsocketClientOutputTransport(BaseOutputTransport):
-    def __init__(self, session: WebsocketClientSession, params: WebsocketClientParams):
+    def __init__(
+        self,
+        transport: BaseTransport,
+        session: WebsocketClientSession,
+        params: WebsocketClientParams,
+    ):
         super().__init__(params)
 
+        self._transport = transport
         self._session = session
         self._params = params
 
-        self._send_interval = (self._audio_chunk_size / self._params.audio_out_sample_rate) / 2
+        # write_raw_audio_frames() is called quickly, as soon as we get audio
+        # (e.g. from the TTS), and since this is just a network connection we
+        # would be sending it to quickly. Instead, we want to block to emulate
+        # an audio device, this is what the send interval is. It will be
+        # computed on StartFrame.
+        self._send_interval = 0
         self._next_send_time = 0
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+        self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
+        await self._params.serializer.setup(frame)
         await self._session.setup(frame)
         await self._session.connect()
 
@@ -170,13 +194,17 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
         await super().cancel(frame)
         await self._session.disconnect()
 
+    async def cleanup(self):
+        await super().cleanup()
+        await self._transport.cleanup()
+
     async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
         await self._write_frame(frame)
 
     async def write_raw_audio_frames(self, frames: bytes):
         frame = OutputAudioRawFrame(
             audio=frames,
-            sample_rate=self._params.audio_out_sample_rate,
+            sample_rate=self.sample_rate,
             num_channels=self._params.audio_out_channels,
         )
 
@@ -232,8 +260,8 @@ class WebsocketClientTransport(BaseTransport):
         )
 
         self._session = WebsocketClientSession(uri, params, callbacks, self.name)
-        self._input: WebsocketClientInputTransport | None = None
-        self._output: WebsocketClientOutputTransport | None = None
+        self._input: Optional[WebsocketClientInputTransport] = None
+        self._output: Optional[WebsocketClientOutputTransport] = None
 
         # Register supported handlers. The user will only be able to register
         # these handlers.
@@ -242,12 +270,12 @@ class WebsocketClientTransport(BaseTransport):
 
     def input(self) -> WebsocketClientInputTransport:
         if not self._input:
-            self._input = WebsocketClientInputTransport(self._session, self._params)
+            self._input = WebsocketClientInputTransport(self, self._session, self._params)
         return self._input
 
     def output(self) -> WebsocketClientOutputTransport:
         if not self._output:
-            self._output = WebsocketClientOutputTransport(self._session, self._params)
+            self._output = WebsocketClientOutputTransport(self, self._session, self._params)
         return self._output
 
     async def _on_connected(self, websocket):

src/pipecat/transports/network/websocket_server.py

@@ -6,9 +6,10 @@
 
 import asyncio
 import io
+import json
 import time
 import wave
-from typing import Awaitable, Callable
+from typing import Awaitable, Callable, Optional
 
 from loguru import logger
 from pydantic import BaseModel
@@ -21,10 +22,11 @@ from pipecat.frames.frames import (
     OutputAudioRawFrame,
     StartFrame,
     StartInterruptionFrame,
+    TransportMessageFrame,
+    TransportMessageUrgentFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.serializers.base_serializer import FrameSerializer
-from pipecat.serializers.protobuf import ProtobufFrameSerializer
 from pipecat.transports.base_input import BaseInputTransport
 from pipecat.transports.base_output import BaseOutputTransport
 from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -39,19 +41,21 @@ except ModuleNotFoundError as e:
 
 class WebsocketServerParams(TransportParams):
     add_wav_header: bool = False
-    serializer: FrameSerializer = ProtobufFrameSerializer()
-    session_timeout: int | None = None
+    serializer: FrameSerializer
+    session_timeout: Optional[int] = None
 
 
 class WebsocketServerCallbacks(BaseModel):
     on_client_connected: Callable[[websockets.WebSocketServerProtocol], Awaitable[None]]
     on_client_disconnected: Callable[[websockets.WebSocketServerProtocol], Awaitable[None]]
     on_session_timeout: Callable[[websockets.WebSocketServerProtocol], Awaitable[None]]
+    on_websocket_ready: Callable[[], Awaitable[None]]
 
 
 class WebsocketServerInputTransport(BaseInputTransport):
     def __init__(
         self,
+        transport: BaseTransport,
         host: str,
         port: int,
         params: WebsocketServerParams,
@@ -60,31 +64,54 @@ class WebsocketServerInputTransport(BaseInputTransport):
     ):
         super().__init__(params, **kwargs)
 
+        self._transport = transport
         self._host = host
         self._port = port
         self._params = params
         self._callbacks = callbacks
 
-        self._websocket: websockets.WebSocketServerProtocol | None = None
+        self._websocket: Optional[websockets.WebSocketServerProtocol] = None
+
+        self._server_task = None
+
+        # This task will monitor the websocket connection periodically.
+        self._monitor_task = None
 
         self._stop_server_event = asyncio.Event()
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
-        self._server_task = self.create_task(self._server_task_handler())
+        await self._params.serializer.setup(frame)
+        if not self._server_task:
+            self._server_task = self.create_task(self._server_task_handler())
 
     async def stop(self, frame: EndFrame):
         await super().stop(frame)
         self._stop_server_event.set()
-        await self.wait_for_task(self._server_task)
+        if self._monitor_task:
+            await self.cancel_task(self._monitor_task)
+            self._monitor_task = None
+        if self._server_task:
+            await self.wait_for_task(self._server_task)
+            self._server_task = None
 
     async def cancel(self, frame: CancelFrame):
         await super().cancel(frame)
-        await self.cancel_task(self._server_task)
+        if self._monitor_task:
+            await self.cancel_task(self._monitor_task)
+            self._monitor_task = None
+        if self._server_task:
+            await self.cancel_task(self._server_task)
+            self._server_task = None
+
+    async def cleanup(self):
+        await super().cleanup()
+        await self._transport.cleanup()
 
     async def _server_task_handler(self):
         logger.info(f"Starting websocket server on {self._host}:{self._port}")
         async with websockets.serve(self._client_handler, self._host, self._port) as server:
+            await self._callbacks.on_websocket_ready()
             await self._stop_server_event.wait()
 
     async def _client_handler(self, websocket: websockets.WebSocketServerProtocol, path):
@@ -99,8 +126,10 @@ class WebsocketServerInputTransport(BaseInputTransport):
         await self._callbacks.on_client_connected(websocket)
 
         # Create a task to monitor the websocket connection
-        if self._params.session_timeout:
-            self.create_task(self._monitor_websocket(websocket))
+        if not self._monitor_task and self._params.session_timeout:
+            self._monitor_task = self.create_task(
+                self._monitor_websocket(websocket, self._params.session_timeout)
+            )
 
         # Handle incoming messages
         try:
@@ -115,7 +144,7 @@ class WebsocketServerInputTransport(BaseInputTransport):
                 else:
                     await self.push_frame(frame)
         except Exception as e:
-            logger.error(f"{self} exception receiving data (class: {e.__class__.__name__})")
+            logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
 
         # Notify disconnection
         await self._callbacks.on_client_disconnected(websocket)
@@ -125,10 +154,13 @@ class WebsocketServerInputTransport(BaseInputTransport):
 
         logger.info(f"Client {websocket.remote_address} disconnected")
 
-    async def _monitor_websocket(self, websocket: websockets.WebSocketServerProtocol):
-        """Wait for self._params.session_timeout seconds, if the websocket is still open, trigger timeout event."""
+    async def _monitor_websocket(
+        self, websocket: websockets.WebSocketServerProtocol, session_timeout: int
+    ):
+        """Wait for session_timeout seconds, if the websocket is still open,
+        trigger timeout event."""
         try:
-            await asyncio.sleep(self._params.session_timeout)
+            await asyncio.sleep(session_timeout)
             if not websocket.closed:
                 await self._callbacks.on_session_timeout(websocket)
         except asyncio.CancelledError:
@@ -137,22 +169,37 @@ class WebsocketServerInputTransport(BaseInputTransport):
 
 
 class WebsocketServerOutputTransport(BaseOutputTransport):
-    def __init__(self, params: WebsocketServerParams, **kwargs):
+    def __init__(self, transport: BaseTransport, params: WebsocketServerParams, **kwargs):
         super().__init__(params, **kwargs)
 
+        self._transport = transport
         self._params = params
 
-        self._websocket: websockets.WebSocketServerProtocol | None = None
+        self._websocket: Optional[websockets.WebSocketServerProtocol] = None
 
-        self._send_interval = (self._audio_chunk_size / self._params.audio_out_sample_rate) / 2
+        # write_raw_audio_frames() is called quickly, as soon as we get audio
+        # (e.g. from the TTS), and since this is just a network connection we
+        # would be sending it to quickly. Instead, we want to block to emulate
+        # an audio device, this is what the send interval is. It will be
+        # computed on StartFrame.
+        self._send_interval = 0
         self._next_send_time = 0
 
-    async def set_client_connection(self, websocket: websockets.WebSocketServerProtocol | None):
+    async def set_client_connection(self, websocket: Optional[websockets.WebSocketServerProtocol]):
         if self._websocket:
             await self._websocket.close()
             logger.warning("Only one client allowed, using new connection")
         self._websocket = websocket
 
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+        await self._params.serializer.setup(frame)
+        self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
+
+    async def cleanup(self):
+        await super().cleanup()
+        await self._transport.cleanup()
+
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
 
@@ -160,6 +207,9 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
             await self._write_frame(frame)
             self._next_send_time = 0
 
+    async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
+        await self._write_frame(frame)
+
     async def write_raw_audio_frames(self, frames: bytes):
         if not self._websocket:
             # Simulate audio playback with a sleep.
@@ -168,7 +218,7 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
 
         frame = OutputAudioRawFrame(
             audio=frames,
-            sample_rate=self._params.audio_out_sample_rate,
+            sample_rate=self.sample_rate,
             num_channels=self._params.audio_out_channels,
         )
 
@@ -197,7 +247,7 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
             if payload and self._websocket:
                 await self._websocket.send(payload)
         except Exception as e:
-            logger.error(f"{self} exception sending data (class: {e.__class__.__name__})")
+            logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
 
     async def _write_audio_sleep(self):
         # Simulate a clock.
@@ -213,14 +263,13 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
 class WebsocketServerTransport(BaseTransport):
     def __init__(
         self,
+        params: WebsocketServerParams,
         host: str = "localhost",
         port: int = 8765,
-        params: WebsocketServerParams = WebsocketServerParams(),
-        input_name: str | None = None,
-        output_name: str | None = None,
-        loop: asyncio.AbstractEventLoop | None = None,
+        input_name: Optional[str] = None,
+        output_name: Optional[str] = None,
     ):
-        super().__init__(input_name=input_name, output_name=output_name, loop=loop)
+        super().__init__(input_name=input_name, output_name=output_name)
         self._host = host
         self._port = port
         self._params = params
@@ -229,27 +278,31 @@ class WebsocketServerTransport(BaseTransport):
             on_client_connected=self._on_client_connected,
             on_client_disconnected=self._on_client_disconnected,
             on_session_timeout=self._on_session_timeout,
+            on_websocket_ready=self._on_websocket_ready,
         )
-        self._input: WebsocketServerInputTransport | None = None
-        self._output: WebsocketServerOutputTransport | None = None
-        self._websocket: websockets.WebSocketServerProtocol | None = None
+        self._input: Optional[WebsocketServerInputTransport] = None
+        self._output: Optional[WebsocketServerOutputTransport] = None
+        self._websocket: Optional[websockets.WebSocketServerProtocol] = None
 
         # Register supported handlers. The user will only be able to register
         # these handlers.
         self._register_event_handler("on_client_connected")
         self._register_event_handler("on_client_disconnected")
         self._register_event_handler("on_session_timeout")
+        self._register_event_handler("on_websocket_ready")
 
     def input(self) -> WebsocketServerInputTransport:
         if not self._input:
             self._input = WebsocketServerInputTransport(
-                self._host, self._port, self._params, self._callbacks, name=self._input_name
+                self, self._host, self._port, self._params, self._callbacks, name=self._input_name
             )
         return self._input
 
     def output(self) -> WebsocketServerOutputTransport:
         if not self._output:
-            self._output = WebsocketServerOutputTransport(self._params, name=self._output_name)
+            self._output = WebsocketServerOutputTransport(
+                self, self._params, name=self._output_name
+            )
         return self._output
 
     async def _on_client_connected(self, websocket):
@@ -268,3 +321,6 @@ class WebsocketServerTransport(BaseTransport):
 
     async def _on_session_timeout(self, websocket):
         await self._call_event_handler("on_session_timeout", websocket)
+
+    async def _on_websocket_ready(self):
+        await self._call_event_handler("on_websocket_ready")

src/pipecat/transports/services/daily.py

@@ -6,22 +6,18 @@
 
 import asyncio
 import time
-import warnings
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from typing import Any, Awaitable, Callable, Mapping, Optional
 
 import aiohttp
 from daily import (
-    CallClient,
-    Daily,
-    EventHandler,
     VirtualCameraDevice,
     VirtualMicrophoneDevice,
     VirtualSpeakerDevice,
 )
 from loguru import logger
-from pydantic import BaseModel, model_validator
+from pydantic import BaseModel
 
 from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams
 from pipecat.frames.frames import (
@@ -46,7 +42,7 @@ from pipecat.transcriptions.language import Language
 from pipecat.transports.base_input import BaseInputTransport
 from pipecat.transports.base_output import BaseOutputTransport
 from pipecat.transports.base_transport import BaseTransport, TransportParams
-from pipecat.utils.asyncio import TaskManager
+from pipecat.utils.asyncio import BaseTaskManager
 
 try:
     from daily import CallClient, Daily, EventHandler
@@ -62,20 +58,41 @@ VAD_RESET_PERIOD_MS = 2000
 
 @dataclass
 class DailyTransportMessageFrame(TransportMessageFrame):
-    participant_id: str | None = None
+    """Frame for transport messages in Daily calls.
+
+    Attributes:
+        participant_id: Optional ID of the participant this message is for/from.
+    """
+
+    participant_id: Optional[str] = None
 
 
 @dataclass
 class DailyTransportMessageUrgentFrame(TransportMessageUrgentFrame):
-    participant_id: str | None = None
+    """Frame for urgent transport messages in Daily calls.
+
+    Attributes:
+        participant_id: Optional ID of the participant this message is for/from.
+    """
+
+    participant_id: Optional[str] = None
 
 
 class WebRTCVADAnalyzer(VADAnalyzer):
-    def __init__(self, *, sample_rate=16000, num_channels=1, params: VADParams = VADParams()):
-        super().__init__(sample_rate=sample_rate, num_channels=num_channels, params=params)
+    """Voice Activity Detection analyzer using WebRTC.
+
+    Implements voice activity detection using Daily's native WebRTC VAD.
+
+    Args:
+        sample_rate: Audio sample rate in Hz.
+        params: VAD configuration parameters (VADParams).
+    """
+
+    def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams = VADParams()):
+        super().__init__(sample_rate=sample_rate, params=params)
 
         self._webrtc_vad = Daily.create_native_vad(
-            reset_period_ms=VAD_RESET_PERIOD_MS, sample_rate=sample_rate, channels=num_channels
+            reset_period_ms=VAD_RESET_PERIOD_MS, sample_rate=self.sample_rate, channels=1
         )
         logger.debug("Loaded native WebRTC VAD")
 
@@ -90,13 +107,32 @@ class WebRTCVADAnalyzer(VADAnalyzer):
 
 
 class DailyDialinSettings(BaseModel):
+    """Settings for Daily's dial-in functionality.
+
+    Attributes:
+        call_id: CallId is represented by UUID and represents the sessionId in the SIP Network.
+        call_domain: Call Domain is represented by UUID and represents your Daily Domain on the SIP Network.
+    """
+
     call_id: str = ""
     call_domain: str = ""
 
 
 class DailyTranscriptionSettings(BaseModel):
+    """Configuration settings for Daily's transcription service.
+
+    Attributes:
+        language: ISO language code for transcription (e.g. "en").
+        model: Transcription model to use (e.g. "nova-2-general").
+        profanity_filter: Whether to filter profanity from transcripts.
+        redact: Whether to redact sensitive information.
+        endpointing: Whether to use endpointing to determine speech segments.
+        punctuate: Whether to add punctuation to transcripts.
+        includeRawResponse: Whether to include raw response data.
+        extra: Additional parameters passed to the Deepgram transcription service.
+    """
+
     language: str = "en"
-    tier: Optional[str] = None
     model: str = "nova-2-general"
     profanity_filter: bool = True
     redact: bool = False
@@ -105,18 +141,18 @@ class DailyTranscriptionSettings(BaseModel):
     includeRawResponse: bool = True
     extra: Mapping[str, Any] = {"interim_results": True}
 
-    @model_validator(mode="before")
-    def check_deprecated_fields(cls, values):
-        with warnings.catch_warnings():
-            warnings.simplefilter("always")
-            if "tier" in values:
-                warnings.warn(
-                    "Field 'tier' is deprecated, use 'model' instead.", DeprecationWarning
-                )
-        return values
-
 
 class DailyParams(TransportParams):
+    """Configuration parameters for Daily transport.
+
+    Args:
+        api_url: Daily API base URL
+        api_key: Daily API authentication key
+        dialin_settings: Optional settings for dial-in functionality
+        transcription_enabled: Whether to enable speech transcription
+        transcription_settings: Configuration for transcription service
+    """
+
     api_url: str = "https://api.daily.co/v1"
     api_key: str = ""
     dialin_settings: Optional[DailyDialinSettings] = None
@@ -125,6 +161,33 @@ class DailyParams(TransportParams):
 
 
 class DailyCallbacks(BaseModel):
+    """Callback handlers for Daily events.
+
+    Attributes:
+        on_joined: Called when bot successfully joined a room.
+        on_left: Called when bot left a room.
+        on_error: Called when an error occurs.
+        on_app_message: Called when receiving an app message.
+        on_call_state_updated: Called when call state changes.
+        on_dialin_connected: Called when dial-in is connected.
+        on_dialin_ready: Called when dial-in is ready.
+        on_dialin_stopped: Called when dial-in is stopped.
+        on_dialin_error: Called when dial-in encounters an error.
+        on_dialin_warning: Called when dial-in has a warning.
+        on_dialout_answered: Called when dial-out is answered.
+        on_dialout_connected: Called when dial-out is connected.
+        on_dialout_stopped: Called when dial-out is stopped.
+        on_dialout_error: Called when dial-out encounters an error.
+        on_dialout_warning: Called when dial-out has a warning.
+        on_participant_joined: Called when a participant joins.
+        on_participant_left: Called when a participant leaves.
+        on_participant_updated: Called when participant info is updated.
+        on_transcription_message: Called when receiving transcription.
+        on_recording_started: Called when recording starts.
+        on_recording_stopped: Called when recording stops.
+        on_recording_error: Called when recording encounters an error.
+    """
+
     on_joined: Callable[[Mapping[str, Any]], Awaitable[None]]
     on_left: Callable[[], Awaitable[None]]
     on_error: Callable[[str], Awaitable[None]]
@@ -166,6 +229,19 @@ def completion_callback(future):
 
 
 class DailyTransportClient(EventHandler):
+    """Core client for interacting with Daily's API.
+
+    Manages the connection to Daily rooms and handles all low-level API interactions.
+
+    Args:
+        room_url: URL of the Daily room to connect to.
+        token: Optional authentication token for the room.
+        bot_name: Display name for the bot in the call.
+        params: Configuration parameters (DailyParams).
+        callbacks: Event callback handlers (DailyCallbacks).
+        transport_name: Name identifier for the transport.
+    """
+
     _daily_initialized: bool = False
 
     # This is necessary to override EventHandler's __new__ method.
@@ -175,7 +251,7 @@ class DailyTransportClient(EventHandler):
     def __init__(
         self,
         room_url: str,
-        token: str | None,
+        token: Optional[str],
         bot_name: str,
         params: DailyParams,
         callbacks: DailyCallbacks,
@@ -188,7 +264,7 @@ class DailyTransportClient(EventHandler):
             Daily.init()
 
         self._room_url: str = room_url
-        self._token: str | None = token
+        self._token: Optional[str] = token
         self._bot_name: str = bot_name
         self._params: DailyParams = params
         self._callbacks = callbacks
@@ -199,11 +275,12 @@ class DailyTransportClient(EventHandler):
         self._transcription_ids = []
         self._transcription_status = None
 
+        self._joining = False
         self._joined = False
         self._joined_event = asyncio.Event()
         self._leave_counter = 0
 
-        self._task_manager: Optional[TaskManager] = None
+        self._task_manager: Optional[BaseTaskManager] = None
 
         # We use the executor to cleanup the client. We just do it from one
         # place, so only one thread is really needed.
@@ -222,33 +299,13 @@ class DailyTransportClient(EventHandler):
         self._callback_queue = asyncio.Queue()
         self._callback_task = None
 
-        self._camera: VirtualCameraDevice | None = None
-        if self._params.camera_out_enabled:
-            self._camera = Daily.create_camera_device(
-                self._camera_name(),
-                width=self._params.camera_out_width,
-                height=self._params.camera_out_height,
-                color_format=self._params.camera_out_color_format,
-            )
+        # Input and ouput sample rates. They will be initialize on setup().
+        self._in_sample_rate = 0
+        self._out_sample_rate = 0
 
-        self._mic: VirtualMicrophoneDevice | None = None
-        if self._params.audio_out_enabled:
-            self._mic = Daily.create_microphone_device(
-                self._mic_name(),
-                sample_rate=self._params.audio_out_sample_rate,
-                channels=self._params.audio_out_channels,
-                non_blocking=True,
-            )
-
-        self._speaker: VirtualSpeakerDevice | None = None
-        if self._params.audio_in_enabled or self._params.vad_enabled:
-            self._speaker = Daily.create_speaker_device(
-                self._speaker_name(),
-                sample_rate=self._params.audio_in_sample_rate,
-                channels=self._params.audio_in_channels,
-                non_blocking=True,
-            )
-            Daily.select_speaker_device(self._speaker_name())
+        self._camera: Optional[VirtualCameraDevice] = None
+        self._mic: Optional[VirtualMicrophoneDevice] = None
+        self._speaker: Optional[VirtualSpeakerDevice] = None
 
     def _camera_name(self):
         return f"camera-{self}"
@@ -259,6 +316,10 @@ class DailyTransportClient(EventHandler):
     def _speaker_name(self):
         return f"speaker-{self}"
 
+    @property
+    def room_url(self) -> str:
+        return self._room_url
+
     @property
     def participant_id(self) -> str:
         return self._participant_id
@@ -277,11 +338,11 @@ class DailyTransportClient(EventHandler):
         )
         await future
 
-    async def read_next_audio_frame(self) -> InputAudioRawFrame | None:
+    async def read_next_audio_frame(self) -> Optional[InputAudioRawFrame]:
         if not self._speaker:
             return None
 
-        sample_rate = self._params.audio_in_sample_rate
+        sample_rate = self._in_sample_rate
         num_channels = self._params.audio_in_channels
         num_frames = int(sample_rate / 100) * 2  # 20ms of audio
 
@@ -315,6 +376,34 @@ class DailyTransportClient(EventHandler):
         self._camera.write_frame(frame.image)
 
     async def setup(self, frame: StartFrame):
+        self._in_sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
+        self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
+
+        if self._params.camera_out_enabled and not self._camera:
+            self._camera = Daily.create_camera_device(
+                self._camera_name(),
+                width=self._params.camera_out_width,
+                height=self._params.camera_out_height,
+                color_format=self._params.camera_out_color_format,
+            )
+
+        if self._params.audio_out_enabled and not self._mic:
+            self._mic = Daily.create_microphone_device(
+                self._mic_name(),
+                sample_rate=self._out_sample_rate,
+                channels=self._params.audio_out_channels,
+                non_blocking=True,
+            )
+
+        if (self._params.audio_in_enabled or self._params.vad_enabled) and not self._speaker:
+            self._speaker = Daily.create_speaker_device(
+                self._speaker_name(),
+                sample_rate=self._in_sample_rate,
+                channels=self._params.audio_in_channels,
+                non_blocking=True,
+            )
+            Daily.select_speaker_device(self._speaker_name())
+
         if not self._task_manager:
             self._task_manager = frame.task_manager
             self._callback_task = self._task_manager.create_task(
@@ -323,13 +412,14 @@ class DailyTransportClient(EventHandler):
             )
 
     async def join(self):
-        # Transport already joined, ignore.
-        if self._joined:
+        # Transport already joined or joining, ignore.
+        if self._joined or self._joining:
             # Increment leave counter if we already joined.
             self._leave_counter += 1
             return
 
         logger.info(f"Joining {self._room_url}")
+        self._joining = True
 
         # For performance reasons, never subscribe to video streams (unless a
         # video renderer is registered).
@@ -344,6 +434,7 @@ class DailyTransportClient(EventHandler):
 
             if not error:
                 self._joined = True
+                self._joining = False
                 # Increment leave counter if we successfully joined.
                 self._leave_counter += 1
 
@@ -362,6 +453,7 @@ class DailyTransportClient(EventHandler):
         except asyncio.TimeoutError:
             error_msg = f"Time out joining {self._room_url}"
             logger.error(error_msg)
+            self._joining = False
             await self._callbacks.on_error(error_msg)
 
     async def _start_transcription(self):
@@ -534,7 +626,7 @@ class DailyTransportClient(EventHandler):
         self._client.stop_recording(stream_id, completion=completion_callback(future))
         await future
 
-    async def send_prebuilt_chat_message(self, message: str, user_name: str | None = None):
+    async def send_prebuilt_chat_message(self, message: str, user_name: Optional[str] = None):
         if not self._joined:
             return
 
@@ -590,6 +682,13 @@ class DailyTransportClient(EventHandler):
         )
         await future
 
+    async def update_remote_participants(self, remote_participants: Mapping[str, Any] = None):
+        future = self._get_event_loop().create_future()
+        self._client.update_remote_participants(
+            remote_participants=remote_participants, completion=completion_callback(future)
+        )
+        await future
+
     #
     #
     # Daily (EventHandler)
@@ -703,35 +802,68 @@ class DailyTransportClient(EventHandler):
 
 
 class DailyInputTransport(BaseInputTransport):
-    def __init__(self, client: DailyTransportClient, params: DailyParams, **kwargs):
+    """Handles incoming media streams and events from Daily calls.
+
+    Processes incoming audio, video, transcriptions and other events from Daily.
+
+    Args:
+        client: DailyTransportClient instance.
+        params: Configuration parameters.
+    """
+
+    def __init__(
+        self,
+        transport: BaseTransport,
+        client: DailyTransportClient,
+        params: DailyParams,
+        **kwargs,
+    ):
         super().__init__(params, **kwargs)
 
+        self._transport = transport
         self._client = client
+        self._params = params
 
         self._video_renderers = {}
 
+        # Whether we have seen a StartFrame already.
+        self._initialized = False
+
         # Task that gets audio data from a device or the network and queues it
         # internally to be processed.
         self._audio_in_task = None
 
-        self._vad_analyzer: VADAnalyzer | None = params.vad_analyzer
-        if params.vad_enabled and not params.vad_analyzer:
-            self._vad_analyzer = WebRTCVADAnalyzer(
-                sample_rate=self._params.audio_in_sample_rate,
-                num_channels=self._params.audio_in_channels,
-            )
+        self._vad_analyzer: Optional[VADAnalyzer] = params.vad_analyzer
+
+    @property
+    def vad_analyzer(self) -> Optional[VADAnalyzer]:
+        return self._vad_analyzer
+
+    def start_audio_in_streaming(self):
+        # Create audio task. It reads audio frames from Daily and push them
+        # internally for VAD processing.
+        if not self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
+            logger.debug(f"Start receiving audio")
+            self._audio_in_task = self.create_task(self._audio_in_task_handler())
 
     async def start(self, frame: StartFrame):
         # Parent start.
         await super().start(frame)
+
+        if self._initialized:
+            return
+
+        self._initialized = True
+
         # Setup client.
         await self._client.setup(frame)
         # Join the room.
         await self._client.join()
-        # Create audio task. It reads audio frames from Daily and push them
-        # internally for VAD processing.
-        if self._params.audio_in_enabled or self._params.vad_enabled:
-            self._audio_in_task = self.create_task(self._audio_in_task_handler())
+        # Inialize WebRTC VAD if needed.
+        if self._params.vad_enabled and not self._params.vad_analyzer:
+            self._vad_analyzer = WebRTCVADAnalyzer(sample_rate=self.sample_rate)
+        if self._params.audio_in_stream_on_start:
+            self.start_audio_in_streaming()
 
     async def stop(self, frame: EndFrame):
         # Parent stop.
@@ -756,9 +888,7 @@ class DailyInputTransport(BaseInputTransport):
     async def cleanup(self):
         await super().cleanup()
         await self._client.cleanup()
-
-    def vad_analyzer(self) -> VADAnalyzer | None:
-        return self._vad_analyzer
+        await self._transport.cleanup()
 
     #
     # FrameProcessor
@@ -768,7 +898,7 @@ class DailyInputTransport(BaseInputTransport):
         await super().process_frame(frame, direction)
 
         if isinstance(frame, UserImageRequestFrame):
-            await self.request_participant_image(frame.user_id)
+            await self.request_participant_image(frame)
 
     #
     # Frames
@@ -805,16 +935,16 @@ class DailyInputTransport(BaseInputTransport):
         self._video_renderers[participant_id] = {
             "framerate": framerate,
             "timestamp": 0,
-            "render_next_frame": False,
+            "render_next_frame": [],
         }
 
         await self._client.capture_participant_video(
             participant_id, self._on_participant_video_frame, framerate, video_source, color_format
         )
 
-    async def request_participant_image(self, participant_id: str):
-        if participant_id in self._video_renderers:
-            self._video_renderers[participant_id]["render_next_frame"] = True
+    async def request_participant_image(self, frame: UserImageRequestFrame):
+        if frame.user_id in self._video_renderers:
+            self._video_renderers[frame.user_id]["render_next_frame"].append(frame)
 
     async def _on_participant_video_frame(self, participant_id: str, buffer, size, format):
         render_frame = False
@@ -823,31 +953,59 @@ class DailyInputTransport(BaseInputTransport):
         prev_time = self._video_renderers[participant_id]["timestamp"]
         framerate = self._video_renderers[participant_id]["framerate"]
 
+        # Some times we render frames because of a request.
+        request_frame = None
+
         if framerate > 0:
             next_time = prev_time + 1 / framerate
             render_frame = (next_time - curr_time) < 0.1
 
         elif self._video_renderers[participant_id]["render_next_frame"]:
-            self._video_renderers[participant_id]["render_next_frame"] = False
+            request_frame = self._video_renderers[participant_id]["render_next_frame"].pop(0)
             render_frame = True
 
         if render_frame:
             frame = UserImageRawFrame(
-                user_id=participant_id, image=buffer, size=size, format=format
+                user_id=participant_id,
+                request=request_frame,
+                image=buffer,
+                size=size,
+                format=format,
             )
             await self.push_frame(frame)
             self._video_renderers[participant_id]["timestamp"] = curr_time
 
 
 class DailyOutputTransport(BaseOutputTransport):
-    def __init__(self, client: DailyTransportClient, params: DailyParams, **kwargs):
+    """Handles outgoing media streams and events to Daily calls.
+
+    Manages sending audio, video and other data to Daily calls.
+
+    Args:
+        client: DailyTransportClient instance.
+        params: Configuration parameters.
+    """
+
+    def __init__(
+        self, transport: BaseTransport, client: DailyTransportClient, params: DailyParams, **kwargs
+    ):
         super().__init__(params, **kwargs)
 
+        self._transport = transport
         self._client = client
 
+        # Whether we have seen a StartFrame already.
+        self._initialized = False
+
     async def start(self, frame: StartFrame):
         # Parent start.
         await super().start(frame)
+
+        if self._initialized:
+            return
+
+        self._initialized = True
+
         # Setup client.
         await self._client.setup(frame)
         # Join the room.
@@ -868,6 +1026,7 @@ class DailyOutputTransport(BaseOutputTransport):
     async def cleanup(self):
         await super().cleanup()
         await self._client.cleanup()
+        await self._transport.cleanup()
 
     async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
         await self._client.send_message(frame)
@@ -880,14 +1039,28 @@ class DailyOutputTransport(BaseOutputTransport):
 
 
 class DailyTransport(BaseTransport):
+    """Transport implementation for Daily audio and video calls.
+
+    Handles audio/video streaming, transcription, recordings, dial-in,
+        dial-out, and call management through Daily's API.
+
+    Args:
+        room_url: URL of the Daily room to connect to.
+        token: Optional authentication token for the room.
+        bot_name: Display name for the bot in the call.
+        params: Configuration parameters (DailyParams) for the transport.
+        input_name: Optional name for the input transport.
+        output_name: Optional name for the output transport.
+    """
+
     def __init__(
         self,
         room_url: str,
-        token: str | None,
+        token: Optional[str],
         bot_name: str,
         params: DailyParams = DailyParams(),
-        input_name: str | None = None,
-        output_name: str | None = None,
+        input_name: Optional[str] = None,
+        output_name: Optional[str] = None,
     ):
         super().__init__(input_name=input_name, output_name=output_name)
 
@@ -918,8 +1091,8 @@ class DailyTransport(BaseTransport):
         self._params = params
 
         self._client = DailyTransportClient(room_url, token, bot_name, params, callbacks, self.name)
-        self._input: DailyInputTransport | None = None
-        self._output: DailyOutputTransport | None = None
+        self._input: Optional[DailyInputTransport] = None
+        self._output: Optional[DailyOutputTransport] = None
 
         self._other_participant_has_joined = False
 
@@ -955,18 +1128,26 @@ class DailyTransport(BaseTransport):
 
     def input(self) -> DailyInputTransport:
         if not self._input:
-            self._input = DailyInputTransport(self._client, self._params, name=self._input_name)
+            self._input = DailyInputTransport(
+                self, self._client, self._params, name=self._input_name
+            )
         return self._input
 
     def output(self) -> DailyOutputTransport:
         if not self._output:
-            self._output = DailyOutputTransport(self._client, self._params, name=self._output_name)
+            self._output = DailyOutputTransport(
+                self, self._client, self._params, name=self._output_name
+            )
         return self._output
 
     #
     # DailyTransport
     #
 
+    @property
+    def room_url(self) -> str:
+        return self._client.room_url
+
     @property
     def participant_id(self) -> str:
         return self._client.participant_id
@@ -1006,7 +1187,7 @@ class DailyTransport(BaseTransport):
     async def stop_recording(self, stream_id=None):
         await self._client.stop_recording(stream_id)
 
-    async def send_prebuilt_chat_message(self, message: str, user_name: str | None = None):
+    async def send_prebuilt_chat_message(self, message: str, user_name: Optional[str] = None):
         """Sends a chat message to Daily's Prebuilt main room.
 
         Args:
@@ -1035,6 +1216,9 @@ class DailyTransport(BaseTransport):
             participant_settings=participant_settings, profile_settings=profile_settings
         )
 
+    async def update_remote_participants(self, remote_participants: Mapping[str, Any] = None):
+        await self._client.update_remote_participants(remote_participants=remote_participants)
+
     async def _on_joined(self, data):
         await self._call_event_handler("on_joined", data)
 

src/pipecat/transports/services/helpers/daily_rest.py

@@ -195,6 +195,10 @@ class DailyMeetingTokenProperties(BaseModel):
         default=None,
         description="Start cloud recording when the user joins the room. This can be used to always record and archive meetings, for example in a customer support context.",
     )
+    permissions: Optional[dict] = Field(
+        default=None,
+        description="Specifies the initial default permissions for a non-meeting-owner participant joining a call.",
+    )
 
 
 class DailyMeetingTokenParams(BaseModel):

src/pipecat/transports/services/livekit.py

@@ -27,7 +27,7 @@ from pipecat.processors.frame_processor import FrameDirection
 from pipecat.transports.base_input import BaseInputTransport
 from pipecat.transports.base_output import BaseOutputTransport
 from pipecat.transports.base_transport import BaseTransport, TransportParams
-from pipecat.utils.asyncio import TaskManager
+from pipecat.utils.asyncio import BaseTaskManager
 
 try:
     from livekit import rtc
@@ -40,12 +40,12 @@ except ModuleNotFoundError as e:
 
 @dataclass
 class LiveKitTransportMessageFrame(TransportMessageFrame):
-    participant_id: str | None = None
+    participant_id: Optional[str] = None
 
 
 @dataclass
 class LiveKitTransportMessageUrgentFrame(TransportMessageUrgentFrame):
-    participant_id: str | None = None
+    participant_id: Optional[str] = None
 
 
 class LiveKitParams(TransportParams):
@@ -79,16 +79,16 @@ class LiveKitTransportClient:
         self._params = params
         self._callbacks = callbacks
         self._transport_name = transport_name
-        self._room: rtc.Room | None = None
+        self._room: Optional[rtc.Room] = None
         self._participant_id: str = ""
         self._connected = False
         self._disconnect_counter = 0
-        self._audio_source: rtc.AudioSource | None = None
-        self._audio_track: rtc.LocalAudioTrack | None = None
+        self._audio_source: Optional[rtc.AudioSource] = None
+        self._audio_track: Optional[rtc.LocalAudioTrack] = None
         self._audio_tracks = {}
         self._audio_queue = asyncio.Queue()
         self._other_participant_has_joined = False
-        self._task_manager: Optional[TaskManager] = None
+        self._task_manager: Optional[BaseTaskManager] = None
 
     @property
     def participant_id(self) -> str:
@@ -101,6 +101,7 @@ class LiveKitTransportClient:
         return self._room
 
     async def setup(self, frame: StartFrame):
+        self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
         if not self._task_manager:
             self._task_manager = frame.task_manager
             self._room = rtc.Room(loop=self._task_manager.get_event_loop())
@@ -138,7 +139,7 @@ class LiveKitTransportClient:
 
             # Set up audio source and track
             self._audio_source = rtc.AudioSource(
-                self._params.audio_out_sample_rate, self._params.audio_out_channels
+                self._out_sample_rate, self._params.audio_out_channels
             )
             self._audio_track = rtc.LocalAudioTrack.create_audio_track(
                 "pipecat-audio", self._audio_source
@@ -171,7 +172,7 @@ class LiveKitTransportClient:
         logger.info(f"Disconnected from {self._room_name}")
         await self._callbacks.on_disconnected()
 
-    async def send_data(self, data: bytes, participant_id: str | None = None):
+    async def send_data(self, data: bytes, participant_id: Optional[str] = None):
         if not self._connected:
             return
 
@@ -344,18 +345,30 @@ class LiveKitTransportClient:
 
 
 class LiveKitInputTransport(BaseInputTransport):
-    def __init__(self, client: LiveKitTransportClient, params: LiveKitParams, **kwargs):
+    def __init__(
+        self,
+        transport: BaseTransport,
+        client: LiveKitTransportClient,
+        params: LiveKitParams,
+        **kwargs,
+    ):
         super().__init__(params, **kwargs)
+        self._transport = transport
         self._client = client
+
         self._audio_in_task = None
-        self._vad_analyzer: VADAnalyzer | None = params.vad_analyzer
+        self._vad_analyzer: Optional[VADAnalyzer] = params.vad_analyzer
         self._resampler = create_default_resampler()
 
+    @property
+    def vad_analyzer(self) -> Optional[VADAnalyzer]:
+        return self._vad_analyzer
+
     async def start(self, frame: StartFrame):
         await super().start(frame)
         await self._client.setup(frame)
         await self._client.connect()
-        if self._params.audio_in_enabled or self._params.vad_enabled:
+        if not self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
             self._audio_in_task = self.create_task(self._audio_in_task_handler())
         logger.info("LiveKitInputTransport started")
 
@@ -372,8 +385,9 @@ class LiveKitInputTransport(BaseInputTransport):
         if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
             await self.cancel_task(self._audio_in_task)
 
-    def vad_analyzer(self) -> VADAnalyzer | None:
-        return self._vad_analyzer
+    async def cleanup(self):
+        await super().cleanup()
+        await self._transport.cleanup()
 
     async def push_app_message(self, message: Any, sender: str):
         frame = LiveKitTransportMessageUrgentFrame(message=message, participant_id=sender)
@@ -401,19 +415,26 @@ class LiveKitInputTransport(BaseInputTransport):
         audio_frame = audio_frame_event.frame
 
         audio_data = await self._resampler.resample(
-            audio_frame.data.tobytes(), audio_frame.sample_rate, self._params.audio_in_sample_rate
+            audio_frame.data.tobytes(), audio_frame.sample_rate, self.sample_rate
         )
 
         return AudioRawFrame(
             audio=audio_data,
-            sample_rate=self._params.audio_in_sample_rate,
+            sample_rate=self.sample_rate,
             num_channels=audio_frame.num_channels,
         )
 
 
 class LiveKitOutputTransport(BaseOutputTransport):
-    def __init__(self, client: LiveKitTransportClient, params: LiveKitParams, **kwargs):
+    def __init__(
+        self,
+        transport: BaseTransport,
+        client: LiveKitTransportClient,
+        params: LiveKitParams,
+        **kwargs,
+    ):
         super().__init__(params, **kwargs)
+        self._transport = transport
         self._client = client
 
     async def start(self, frame: StartFrame):
@@ -431,6 +452,10 @@ class LiveKitOutputTransport(BaseOutputTransport):
         await super().cancel(frame)
         await self._client.disconnect()
 
+    async def cleanup(self):
+        await super().cleanup()
+        await self._transport.cleanup()
+
     async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
         if isinstance(frame, (LiveKitTransportMessageFrame, LiveKitTransportMessageUrgentFrame)):
             await self._client.send_data(frame.message.encode(), frame.participant_id)
@@ -448,7 +473,7 @@ class LiveKitOutputTransport(BaseOutputTransport):
 
         return rtc.AudioFrame(
             data=pipecat_audio,
-            sample_rate=self._params.audio_out_sample_rate,
+            sample_rate=self.sample_rate,
             num_channels=self._params.audio_out_channels,
             samples_per_channel=samples_per_channel,
         )
@@ -461,8 +486,8 @@ class LiveKitTransport(BaseTransport):
         token: str,
         room_name: str,
         params: LiveKitParams = LiveKitParams(),
-        input_name: str | None = None,
-        output_name: str | None = None,
+        input_name: Optional[str] = None,
+        output_name: Optional[str] = None,
     ):
         super().__init__(input_name=input_name, output_name=output_name)
 
@@ -481,8 +506,8 @@ class LiveKitTransport(BaseTransport):
         self._client = LiveKitTransportClient(
             url, token, room_name, self._params, callbacks, self.name
         )
-        self._input: LiveKitInputTransport | None = None
-        self._output: LiveKitOutputTransport | None = None
+        self._input: Optional[LiveKitInputTransport] = None
+        self._output: Optional[LiveKitOutputTransport] = None
 
         self._register_event_handler("on_connected")
         self._register_event_handler("on_disconnected")
@@ -497,13 +522,15 @@ class LiveKitTransport(BaseTransport):
 
     def input(self) -> LiveKitInputTransport:
         if not self._input:
-            self._input = LiveKitInputTransport(self._client, self._params, name=self._input_name)
+            self._input = LiveKitInputTransport(
+                self, self._client, self._params, name=self._input_name
+            )
         return self._input
 
     def output(self) -> LiveKitOutputTransport:
         if not self._output:
             self._output = LiveKitOutputTransport(
-                self._client, self._params, name=self._output_name
+                self, self._client, self._params, name=self._output_name
             )
         return self._output
 
@@ -560,25 +587,18 @@ class LiveKitTransport(BaseTransport):
             await self._input.push_app_message(data.decode(), participant_id)
         await self._call_event_handler("on_data_received", data, participant_id)
 
-    async def send_message(self, message: str, participant_id: str | None = None):
+    async def send_message(self, message: str, participant_id: Optional[str] = None):
         if self._output:
             frame = LiveKitTransportMessageFrame(message=message, participant_id=participant_id)
             await self._output.send_message(frame)
 
-    async def send_message_urgent(self, message: str, participant_id: str | None = None):
+    async def send_message_urgent(self, message: str, participant_id: Optional[str] = None):
         if self._output:
             frame = LiveKitTransportMessageUrgentFrame(
                 message=message, participant_id=participant_id
             )
             await self._output.send_message(frame)
 
-    async def cleanup(self):
-        if self._input:
-            await self._input.cleanup()
-        if self._output:
-            await self._output.cleanup()
-        await self._client.disconnect()
-
     async def on_room_event(self, event):
         # Handle room events
         pass

src/pipecat/utils/asyncio.py

@@ -5,12 +5,76 @@
 #
 
 import asyncio
+from abc import ABC, abstractmethod
 from typing import Coroutine, Optional, Set
 
 from loguru import logger
 
 
-class TaskManager:
+class BaseTaskManager(ABC):
+    @abstractmethod
+    def set_event_loop(self, loop: asyncio.AbstractEventLoop):
+        pass
+
+    @abstractmethod
+    def get_event_loop(self) -> asyncio.AbstractEventLoop:
+        pass
+
+    @abstractmethod
+    def create_task(self, coroutine: Coroutine, name: str) -> asyncio.Task:
+        """
+        Creates and schedules a new asyncio Task that runs the given coroutine.
+
+        The task is added to a global set of created tasks.
+
+        Args:
+            loop (asyncio.AbstractEventLoop): The event loop to use for creating the task.
+            coroutine (Coroutine): The coroutine to be executed within the task.
+            name (str): The name to assign to the task for identification.
+
+        Returns:
+            asyncio.Task: The created task object.
+        """
+        pass
+
+    @abstractmethod
+    async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None):
+        """Wait for an asyncio.Task to complete with optional timeout handling.
+
+        This function awaits the specified asyncio.Task and handles scenarios for
+        timeouts, cancellations, and other exceptions. It also ensures that the task
+        is removed from the set of registered tasks upon completion or failure.
+
+        Args:
+            task (asyncio.Task): The asyncio Task to wait for.
+            timeout (Optional[float], optional): The maximum number of seconds
+                to wait for the task to complete. If None, waits indefinitely.
+                Defaults to None.
+        """
+        pass
+
+    @abstractmethod
+    async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
+        """Cancels the given asyncio Task and awaits its completion with an
+        optional timeout.
+
+        This function removes the task from the set of registered tasks upon
+        completion or failure.
+
+        Args:
+            task (asyncio.Task): The task to be cancelled.
+            timeout (Optional[float]): The optional timeout in seconds to wait for the task to cancel.
+
+        """
+        pass
+
+    @abstractmethod
+    def current_tasks(self) -> Set[asyncio.Task]:
+        """Returns the list of currently created/registered tasks."""
+        pass
+
+
+class TaskManager(BaseTaskManager):
     def __init__(self) -> None:
         self._tasks: Set[asyncio.Task] = set()
         self._loop: Optional[asyncio.AbstractEventLoop] = None
@@ -80,6 +144,7 @@ class TaskManager:
             logger.warning(f"{name}: timed out waiting for task to finish")
         except asyncio.CancelledError:
             logger.trace(f"{name}: unexpected task cancellation (maybe Ctrl-C?)")
+            raise
         except Exception as e:
             logger.exception(f"{name}: unexpected exception while stopping task: {e}")
         finally:

src/pipecat/utils/base_object.py

@@ -0,0 +1,87 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import inspect
+from abc import ABC
+from typing import Optional
+
+from loguru import logger
+
+from pipecat.utils.utils import obj_count, obj_id
+
+
+class BaseObject(ABC):
+    def __init__(self, *, name: Optional[str] = None):
+        self._id: int = obj_id()
+        self._name = name or f"{self.__class__.__name__}#{obj_count(self)}"
+
+        # Registered event handlers.
+        self._event_handlers: dict = {}
+
+        # Set of tasks being executed. When a task finishes running it gets
+        # automatically removed from the set. When we cleanup we wait for all
+        # event tasks still being executed.
+        self._event_tasks = set()
+
+    @property
+    def id(self) -> int:
+        return self._id
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    async def cleanup(self):
+        if self._event_tasks:
+            event_names, tasks = zip(*self._event_tasks)
+            logger.debug(f"{self} wating on event handlers to finish {list(event_names)}...")
+            await asyncio.wait(tasks)
+
+    def event_handler(self, event_name: str):
+        def decorator(handler):
+            self.add_event_handler(event_name, handler)
+            return handler
+
+        return decorator
+
+    def add_event_handler(self, event_name: str, handler):
+        if event_name not in self._event_handlers:
+            raise Exception(f"Event handler {event_name} not registered")
+        self._event_handlers[event_name].append(handler)
+
+    def _register_event_handler(self, event_name: str):
+        if event_name in self._event_handlers:
+            raise Exception(f"Event handler {event_name} already registered")
+        self._event_handlers[event_name] = []
+
+    async def _call_event_handler(self, event_name: str, *args, **kwargs):
+        # Create the task.
+        task = asyncio.create_task(self._run_task(event_name, *args, **kwargs))
+
+        # Add it to our list of event tasks.
+        self._event_tasks.add((event_name, task))
+
+        # Remove the task from the event tasks list when the task completes.
+        task.add_done_callback(self._event_task_finished)
+
+    async def _run_task(self, event_name: str, *args, **kwargs):
+        try:
+            for handler in self._event_handlers[event_name]:
+                if inspect.iscoroutinefunction(handler):
+                    await handler(self, *args, **kwargs)
+                else:
+                    handler(self, *args, **kwargs)
+        except Exception as e:
+            logger.exception(f"Exception in event handler {event_name}: {e}")
+
+    def _event_task_finished(self, task: asyncio.Task):
+        tuple_to_remove = next((t for t in self._event_tasks if t[1] == task), None)
+        if tuple_to_remove:
+            self._event_tasks.discard(tuple_to_remove)
+
+    def __str__(self):
+        return self.name